1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Machine check handler 4 * 5 * Copyright IBM Corp. 2000, 2009 6 * Author(s): Ingo Adlung <adlung@de.ibm.com>, 7 * Martin Schwidefsky <schwidefsky@de.ibm.com>, 8 * Cornelia Huck <cornelia.huck@de.ibm.com>, 9 */ 10 11#include <linux/kernel_stat.h> 12#include <linux/init.h> 13#include <linux/errno.h> 14#include <linux/entry-common.h> 15#include <linux/hardirq.h> 16#include <linux/log2.h> 17#include <linux/kprobes.h> 18#include <linux/kmemleak.h> 19#include <linux/time.h> 20#include <linux/module.h> 21#include <linux/sched/signal.h> 22#include <linux/kvm_host.h> 23#include <linux/export.h> 24#include <asm/lowcore.h> 25#include <asm/ctlreg.h> 26#include <asm/fpu.h> 27#include <asm/smp.h> 28#include <asm/stp.h> 29#include <asm/cputime.h> 30#include <asm/nmi.h> 31#include <asm/crw.h> 32#include <asm/asm-offsets.h> 33#include <asm/pai.h> 34#include <asm/vtime.h> 35 36struct mcck_struct { 37 unsigned int kill_task : 1; 38 unsigned int channel_report : 1; 39 unsigned int warning : 1; 40 unsigned int stp_queue : 1; 41 unsigned long mcck_code; 42}; 43 44static DEFINE_PER_CPU(struct mcck_struct, cpu_mcck); 45 46static inline int nmi_needs_mcesa(void) 47{ 48 return cpu_has_vx() || MACHINE_HAS_GS; 49} 50 51/* 52 * The initial machine check extended save area for the boot CPU. 53 * It will be replaced on the boot CPU reinit with an allocated 54 * structure. The structure is required for machine check happening 55 * early in the boot process. 56 */ 57static struct mcesa boot_mcesa __aligned(MCESA_MAX_SIZE); 58 59void __init nmi_alloc_mcesa_early(u64 *mcesad) 60{ 61 if (!nmi_needs_mcesa()) 62 return; 63 *mcesad = __pa(&boot_mcesa); 64 if (MACHINE_HAS_GS) 65 *mcesad |= ilog2(MCESA_MAX_SIZE); 66} 67 68int nmi_alloc_mcesa(u64 *mcesad) 69{ 70 unsigned long size; 71 void *origin; 72 73 *mcesad = 0; 74 if (!nmi_needs_mcesa()) 75 return 0; 76 size = MACHINE_HAS_GS ? MCESA_MAX_SIZE : MCESA_MIN_SIZE; 77 origin = kmalloc(size, GFP_KERNEL); 78 if (!origin) 79 return -ENOMEM; 80 /* The pointer is stored with mcesa_bits ORed in */ 81 kmemleak_not_leak(origin); 82 *mcesad = __pa(origin); 83 if (MACHINE_HAS_GS) 84 *mcesad |= ilog2(MCESA_MAX_SIZE); 85 return 0; 86} 87 88void nmi_free_mcesa(u64 *mcesad) 89{ 90 if (!nmi_needs_mcesa()) 91 return; 92 kfree(__va(*mcesad & MCESA_ORIGIN_MASK)); 93} 94 95static __always_inline char *nmi_puts(char *dest, const char *src) 96{ 97 while (*src) 98 *dest++ = *src++; 99 *dest = 0; 100 return dest; 101} 102 103static __always_inline char *u64_to_hex(char *dest, u64 val) 104{ 105 int i, num; 106 107 for (i = 1; i <= 16; i++) { 108 num = (val >> (64 - 4 * i)) & 0xf; 109 if (num >= 10) 110 *dest++ = 'A' + num - 10; 111 else 112 *dest++ = '0' + num; 113 } 114 *dest = 0; 115 return dest; 116} 117 118static notrace void s390_handle_damage(void) 119{ 120 union ctlreg0 cr0, cr0_new; 121 char message[100]; 122 psw_t psw_save; 123 char *ptr; 124 125 smp_emergency_stop(); 126 diag_amode31_ops.diag308_reset(); 127 ptr = nmi_puts(message, "System stopped due to unrecoverable machine check, code: 0x"); 128 u64_to_hex(ptr, S390_lowcore.mcck_interruption_code); 129 130 /* 131 * Disable low address protection and make machine check new PSW a 132 * disabled wait PSW. Any additional machine check cannot be handled. 133 */ 134 local_ctl_store(0, &cr0.reg); 135 cr0_new = cr0; 136 cr0_new.lap = 0; 137 local_ctl_load(0, &cr0_new.reg); 138 psw_save = S390_lowcore.mcck_new_psw; 139 psw_bits(S390_lowcore.mcck_new_psw).io = 0; 140 psw_bits(S390_lowcore.mcck_new_psw).ext = 0; 141 psw_bits(S390_lowcore.mcck_new_psw).wait = 1; 142 sclp_emergency_printk(message); 143 144 /* 145 * Restore machine check new PSW and control register 0 to original 146 * values. This makes possible system dump analysis easier. 147 */ 148 S390_lowcore.mcck_new_psw = psw_save; 149 local_ctl_load(0, &cr0.reg); 150 disabled_wait(); 151 while (1); 152} 153NOKPROBE_SYMBOL(s390_handle_damage); 154 155/* 156 * Main machine check handler function. Will be called with interrupts disabled 157 * and machine checks enabled. 158 */ 159void s390_handle_mcck(void) 160{ 161 struct mcck_struct mcck; 162 unsigned long mflags; 163 164 /* 165 * Disable machine checks and get the current state of accumulated 166 * machine checks. Afterwards delete the old state and enable machine 167 * checks again. 168 */ 169 local_mcck_save(mflags); 170 mcck = *this_cpu_ptr(&cpu_mcck); 171 memset(this_cpu_ptr(&cpu_mcck), 0, sizeof(mcck)); 172 local_mcck_restore(mflags); 173 174 if (mcck.channel_report) 175 crw_handle_channel_report(); 176 /* 177 * A warning may remain for a prolonged period on the bare iron. 178 * (actually until the machine is powered off, or the problem is gone) 179 * So we just stop listening for the WARNING MCH and avoid continuously 180 * being interrupted. One caveat is however, that we must do this per 181 * processor and cannot use the smp version of ctl_clear_bit(). 182 * On VM we only get one interrupt per virtally presented machinecheck. 183 * Though one suffices, we may get one interrupt per (virtual) cpu. 184 */ 185 if (mcck.warning) { /* WARNING pending ? */ 186 static int mchchk_wng_posted = 0; 187 188 /* Use single cpu clear, as we cannot handle smp here. */ 189 local_ctl_clear_bit(14, CR14_WARNING_SUBMASK_BIT); 190 if (xchg(&mchchk_wng_posted, 1) == 0) 191 kill_cad_pid(SIGPWR, 1); 192 } 193 if (mcck.stp_queue) 194 stp_queue_work(); 195 if (mcck.kill_task) { 196 printk(KERN_EMERG "mcck: Terminating task because of machine " 197 "malfunction (code 0x%016lx).\n", mcck.mcck_code); 198 printk(KERN_EMERG "mcck: task: %s, pid: %d.\n", 199 current->comm, current->pid); 200 if (is_global_init(current)) 201 panic("mcck: Attempting to kill init!\n"); 202 do_send_sig_info(SIGKILL, SEND_SIG_PRIV, current, PIDTYPE_PID); 203 } 204} 205 206/** 207 * nmi_registers_valid - verify if registers are valid 208 * @mci: machine check interruption code 209 * 210 * Inspect a machine check interruption code and verify if all required 211 * registers are valid. For some registers the corresponding validity bit is 212 * ignored and the registers are set to the expected value. 213 * Returns true if all registers are valid, otherwise false. 214 */ 215static bool notrace nmi_registers_valid(union mci mci) 216{ 217 union ctlreg2 cr2; 218 219 /* 220 * The getcpu vdso syscall reads the CPU number from the programmable 221 * field of the TOD clock. Disregard the TOD programmable register 222 * validity bit and load the CPU number into the TOD programmable field 223 * unconditionally. 224 */ 225 set_tod_programmable_field(raw_smp_processor_id()); 226 /* 227 * Set the clock comparator register to the next expected value. 228 */ 229 set_clock_comparator(S390_lowcore.clock_comparator); 230 if (!mci.gr || !mci.fp || !mci.fc) 231 return false; 232 /* 233 * The vector validity must only be checked if not running a 234 * KVM guest. For KVM guests the machine check is forwarded by 235 * KVM and it is the responsibility of the guest to take 236 * appropriate actions. The host vector or FPU values have been 237 * saved by KVM and will be restored by KVM. 238 */ 239 if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) 240 return false; 241 if (!mci.ar) 242 return false; 243 /* 244 * Two cases for guarded storage registers: 245 * - machine check in kernel or userspace 246 * - machine check while running SIE (KVM guest) 247 * For kernel or userspace the userspace values of guarded storage 248 * control can not be recreated, the process must be terminated. 249 * For SIE the guest values of guarded storage can not be recreated. 250 * This is either due to a bug or due to GS being disabled in the 251 * guest. The guest will be notified by KVM code and the guests machine 252 * check handling must take care of this. The host values are saved by 253 * KVM and are not affected. 254 */ 255 cr2.reg = S390_lowcore.cregs_save_area[2]; 256 if (cr2.gse && !mci.gs && !test_cpu_flag(CIF_MCCK_GUEST)) 257 return false; 258 if (!mci.ms || !mci.pm || !mci.ia) 259 return false; 260 return true; 261} 262NOKPROBE_SYMBOL(nmi_registers_valid); 263 264/* 265 * Backup the guest's machine check info to its description block 266 */ 267static void notrace s390_backup_mcck_info(struct pt_regs *regs) 268{ 269 struct mcck_volatile_info *mcck_backup; 270 struct sie_page *sie_page; 271 272 /* r14 contains the sie block, which was set in sie64a */ 273 struct kvm_s390_sie_block *sie_block = phys_to_virt(regs->gprs[14]); 274 275 if (sie_block == NULL) 276 /* Something's seriously wrong, stop system. */ 277 s390_handle_damage(); 278 279 sie_page = container_of(sie_block, struct sie_page, sie_block); 280 mcck_backup = &sie_page->mcck_info; 281 mcck_backup->mcic = S390_lowcore.mcck_interruption_code & 282 ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); 283 mcck_backup->ext_damage_code = S390_lowcore.external_damage_code; 284 mcck_backup->failing_storage_address 285 = S390_lowcore.failing_storage_address; 286} 287NOKPROBE_SYMBOL(s390_backup_mcck_info); 288 289#define MAX_IPD_COUNT 29 290#define MAX_IPD_TIME (5 * 60 * USEC_PER_SEC) /* 5 minutes */ 291 292#define ED_STP_ISLAND 6 /* External damage STP island check */ 293#define ED_STP_SYNC 7 /* External damage STP sync check */ 294 295#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) 296 297/* 298 * machine check handler. 299 */ 300void notrace s390_do_machine_check(struct pt_regs *regs) 301{ 302 static int ipd_count; 303 static DEFINE_SPINLOCK(ipd_lock); 304 static unsigned long long last_ipd; 305 struct mcck_struct *mcck; 306 unsigned long long tmp; 307 irqentry_state_t irq_state; 308 union mci mci; 309 unsigned long mcck_dam_code; 310 int mcck_pending = 0; 311 312 irq_state = irqentry_nmi_enter(regs); 313 314 if (user_mode(regs)) 315 update_timer_mcck(); 316 inc_irq_stat(NMI_NMI); 317 mci.val = S390_lowcore.mcck_interruption_code; 318 mcck = this_cpu_ptr(&cpu_mcck); 319 320 /* 321 * Reinject the instruction processing damages' machine checks 322 * including Delayed Access Exception into the guest 323 * instead of damaging the host if they happen in the guest. 324 */ 325 if (mci.pd && !test_cpu_flag(CIF_MCCK_GUEST)) { 326 if (mci.b) { 327 /* Processing backup -> verify if we can survive this */ 328 u64 z_mcic, o_mcic, t_mcic; 329 z_mcic = (1ULL<<63 | 1ULL<<59 | 1ULL<<29); 330 o_mcic = (1ULL<<43 | 1ULL<<42 | 1ULL<<41 | 1ULL<<40 | 331 1ULL<<36 | 1ULL<<35 | 1ULL<<34 | 1ULL<<32 | 332 1ULL<<30 | 1ULL<<21 | 1ULL<<20 | 1ULL<<17 | 333 1ULL<<16); 334 t_mcic = mci.val; 335 336 if (((t_mcic & z_mcic) != 0) || 337 ((t_mcic & o_mcic) != o_mcic)) { 338 s390_handle_damage(); 339 } 340 341 /* 342 * Nullifying exigent condition, therefore we might 343 * retry this instruction. 344 */ 345 spin_lock(&ipd_lock); 346 tmp = get_tod_clock(); 347 if (((tmp - last_ipd) >> 12) < MAX_IPD_TIME) 348 ipd_count++; 349 else 350 ipd_count = 1; 351 last_ipd = tmp; 352 if (ipd_count == MAX_IPD_COUNT) 353 s390_handle_damage(); 354 spin_unlock(&ipd_lock); 355 } else { 356 /* Processing damage -> stopping machine */ 357 s390_handle_damage(); 358 } 359 } 360 if (!nmi_registers_valid(mci)) { 361 if (!user_mode(regs)) 362 s390_handle_damage(); 363 /* 364 * Couldn't restore all register contents for the 365 * user space process -> mark task for termination. 366 */ 367 mcck->kill_task = 1; 368 mcck->mcck_code = mci.val; 369 mcck_pending = 1; 370 } 371 372 /* 373 * Backup the machine check's info if it happens when the guest 374 * is running. 375 */ 376 if (test_cpu_flag(CIF_MCCK_GUEST)) 377 s390_backup_mcck_info(regs); 378 379 if (mci.cd) { 380 /* Timing facility damage */ 381 s390_handle_damage(); 382 } 383 if (mci.ed && mci.ec) { 384 /* External damage */ 385 if (S390_lowcore.external_damage_code & (1U << ED_STP_SYNC)) 386 mcck->stp_queue |= stp_sync_check(); 387 if (S390_lowcore.external_damage_code & (1U << ED_STP_ISLAND)) 388 mcck->stp_queue |= stp_island_check(); 389 mcck_pending = 1; 390 } 391 /* 392 * Reinject storage related machine checks into the guest if they 393 * happen when the guest is running. 394 */ 395 if (!test_cpu_flag(CIF_MCCK_GUEST)) { 396 /* Storage error uncorrected */ 397 if (mci.se) 398 s390_handle_damage(); 399 /* Storage key-error uncorrected */ 400 if (mci.ke) 401 s390_handle_damage(); 402 /* Storage degradation */ 403 if (mci.ds && mci.fa) 404 s390_handle_damage(); 405 } 406 if (mci.cp) { 407 /* Channel report word pending */ 408 mcck->channel_report = 1; 409 mcck_pending = 1; 410 } 411 if (mci.w) { 412 /* Warning pending */ 413 mcck->warning = 1; 414 mcck_pending = 1; 415 } 416 417 /* 418 * If there are only Channel Report Pending and External Damage 419 * machine checks, they will not be reinjected into the guest 420 * because they refer to host conditions only. 421 */ 422 mcck_dam_code = (mci.val & MCIC_SUBCLASS_MASK); 423 if (test_cpu_flag(CIF_MCCK_GUEST) && 424 (mcck_dam_code & MCCK_CODE_NO_GUEST) != mcck_dam_code) { 425 /* Set exit reason code for host's later handling */ 426 *((long *)(regs->gprs[15] + __SF_SIE_REASON)) = -EINTR; 427 } 428 clear_cpu_flag(CIF_MCCK_GUEST); 429 430 if (mcck_pending) 431 schedule_mcck_handler(); 432 433 irqentry_nmi_exit(regs, irq_state); 434} 435NOKPROBE_SYMBOL(s390_do_machine_check); 436 437static int __init machine_check_init(void) 438{ 439 system_ctl_set_bit(14, CR14_EXTERNAL_DAMAGE_SUBMASK_BIT); 440 system_ctl_set_bit(14, CR14_RECOVERY_SUBMASK_BIT); 441 system_ctl_set_bit(14, CR14_WARNING_SUBMASK_BIT); 442 return 0; 443} 444early_initcall(machine_check_init); 445