diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index fd172c41df90..95813dca24ec 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -9,6 +9,7 @@ #define __ASM_IO_H #include +#include #include #include @@ -24,24 +25,28 @@ #define __raw_writeb __raw_writeb static inline void __raw_writeb(u8 val, volatile void __iomem *addr) { + log_write_mmio(val, 8, addr); asm volatile("strb %w0, [%1]" : : "rZ" (val), "r" (addr)); } #define __raw_writew __raw_writew static inline void __raw_writew(u16 val, volatile void __iomem *addr) { + log_write_mmio(val, 16, addr); asm volatile("strh %w0, [%1]" : : "rZ" (val), "r" (addr)); } #define __raw_writel __raw_writel static __always_inline void __raw_writel(u32 val, volatile void __iomem *addr) { + log_write_mmio(val, 32, addr); asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr)); } #define __raw_writeq __raw_writeq static inline void __raw_writeq(u64 val, volatile void __iomem *addr) { + log_write_mmio(val, 64, addr); asm volatile("str %x0, [%1]" : : "rZ" (val), "r" (addr)); } @@ -49,10 +54,13 @@ static inline void __raw_writeq(u64 val, volatile void __iomem *addr) static inline u8 __raw_readb(const volatile void __iomem *addr) { u8 val; + + log_read_mmio(8, addr); asm volatile(ALTERNATIVE("ldrb %w0, [%1]", "ldarb %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); + log_post_read_mmio(val, 8, addr); return val; } @@ -61,10 +69,12 @@ static inline u16 __raw_readw(const volatile void __iomem *addr) { u16 val; + log_read_mmio(16, addr); asm volatile(ALTERNATIVE("ldrh %w0, [%1]", "ldarh %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); + log_post_read_mmio(val, 16, addr); return val; } @@ -72,10 +82,13 @@ static inline u16 __raw_readw(const volatile void __iomem *addr) static __always_inline u32 __raw_readl(const volatile void __iomem *addr) { u32 val; + + log_read_mmio(32, addr); asm volatile(ALTERNATIVE("ldr %w0, [%1]", "ldar %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); + log_post_read_mmio(val, 32, addr); return val; } @@ -83,10 +96,13 @@ static __always_inline u32 __raw_readl(const volatile void __iomem *addr) static inline u64 __raw_readq(const volatile void __iomem *addr) { u64 val; + + log_read_mmio(64, addr); asm volatile(ALTERNATIVE("ldr %0, [%1]", "ldar %0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); + log_post_read_mmio(val, 64, addr); return val; } diff --git a/include/linux/log_mmiorw.h b/include/linux/log_mmiorw.h new file mode 100644 index 000000000000..b42ef5ac8d99 --- /dev/null +++ b/include/linux/log_mmiorw.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ +#ifndef __LOG_MMIORW_H__ +#define __LOG_MMIORW_H__ + +#include +#include +#include + +/* + * TODO - io.h is included in NVHE files and these tracepoints are getting + * enabled for NVHE too. To avoid these tracepoints enabling in NHVE below + * condition is introduced. + * !(defined(__KVM_NVHE_HYPERVISOR__)) + */ +#if IS_ENABLED(CONFIG_TRACE_MMIO_ACCESS) && !(defined(__KVM_NVHE_HYPERVISOR__)) +DECLARE_TRACEPOINT(rwmmio_write); +DECLARE_TRACEPOINT(rwmmio_read); +DECLARE_TRACEPOINT(rwmmio_post_read); + +void __log_write_mmio(u64 val, u8 width, volatile void __iomem *addr); +void __log_read_mmio(u8 width, const volatile void __iomem *addr); +void __log_post_read_mmio(u64 val, u8 width, const volatile void __iomem *addr); + +#define log_write_mmio(val, width, addr) \ +do { \ + if (tracepoint_enabled(rwmmio_write)) \ + __log_write_mmio(val, width, addr); \ +} while (0) + +#define log_read_mmio(width, addr) \ +do { \ + if (tracepoint_enabled(rwmmio_read)) \ + __log_read_mmio(width, addr); \ +} while (0) + +#define log_post_read_mmio(val, width, addr) \ +do { \ + if (tracepoint_enabled(rwmmio_post_read)) \ + __log_post_read_mmio(val, width, addr); \ +} while (0) + +#else +static inline void log_write_mmio(u64 val, u8 width, volatile void __iomem *addr) +{ } +static inline void log_read_mmio(u8 width, const volatile void __iomem *addr) +{ } +static inline void log_post_read_mmio(u64 val, u8 width, const volatile void __iomem *addr) +{ } +#endif /* CONFIG_TRACE_MMIO_ACCESS */ + +#endif /* __LOG_MMIORW_H__ */ diff --git a/include/trace/events/rwmmio.h b/include/trace/events/rwmmio.h new file mode 100644 index 000000000000..83a475692448 --- /dev/null +++ b/include/trace/events/rwmmio.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rwmmio + +#if !defined(_TRACE_MMIO_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MMIO_H + +#include + +TRACE_EVENT(rwmmio_write, + + TP_PROTO(unsigned long fn, u64 val, u8 width, volatile void __iomem *addr), + + TP_ARGS(fn, val, width, addr), + + TP_STRUCT__entry( + __field(u64, fn) + __field(u64, val) + __field(u8, width) + __field(u64, addr) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->val = val; + __entry->width = width; + __entry->addr = (u64)addr; + ), + + TP_printk("%pS write addr=%llx of width=%x val=0x%llx\n", + __entry->fn, __entry->addr, __entry->width, __entry->val) +); + +TRACE_EVENT(rwmmio_read, + + TP_PROTO(unsigned long fn, u8 width, const volatile void __iomem *addr), + + TP_ARGS(fn, width, addr), + + TP_STRUCT__entry( + __field(u64, fn) + __field(u8, width) + __field(u64, addr) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->width = width; + __entry->addr = (u64)addr; + ), + + TP_printk("%pS read addr=%llx of width=%x\n", + __entry->fn, __entry->addr, __entry->width) +); + +TRACE_EVENT(rwmmio_post_read, + + TP_PROTO(unsigned long fn, u64 val, u8 width, const volatile void __iomem *addr), + + TP_ARGS(fn, val, width, addr), + + TP_STRUCT__entry( + __field(u64, fn) + __field(u64, val) + __field(u8, width) + __field(u64, addr) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->val = val; + __entry->width = width; + __entry->addr = (u64)addr; + ), + + TP_printk("%pS read addr=%llx of width=%x val=0x%llx\n", + __entry->fn, __entry->addr, __entry->width, __entry->val) +); + +#endif /* _TRACE_MMIO_H */ + +#include diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 9b8c849e1285..42303d99ea71 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -86,6 +86,14 @@ config RING_BUFFER_ALLOW_SWAP Allow the use of ring_buffer_swap_cpu. Adds a very slight overhead to tracing when enabled. +config TRACE_MMIO_ACCESS + bool "Register read/write tracing" + depends on TRACING + depends on ARM64 + help + Create tracepoints for IO read/write operations. These trace events + can be used for logging all MMIO read/write operations. + config PREEMPTIRQ_TRACEPOINTS bool depends on TRACE_PREEMPT_TOGGLE || TRACE_IRQFLAGS diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index e153be351548..33e356304964 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -94,5 +94,6 @@ obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o +obj-$(CONFIG_TRACE_MMIO_ACCESS) += trace_readwrite.o libftrace-y := ftrace.o diff --git a/kernel/trace/trace_readwrite.c b/kernel/trace/trace_readwrite.c new file mode 100644 index 000000000000..bc2e8f85537d --- /dev/null +++ b/kernel/trace/trace_readwrite.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Register read and write tracepoints + * + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#ifdef CONFIG_TRACE_MMIO_ACCESS +void __log_write_mmio(u64 val, u8 width, volatile void __iomem *addr) +{ + trace_rwmmio_write(CALLER_ADDR0, val, width, addr); +} +EXPORT_SYMBOL_GPL(__log_write_mmio); +EXPORT_TRACEPOINT_SYMBOL_GPL(rwmmio_write); + +void __log_read_mmio(u8 width, const volatile void __iomem *addr) +{ + trace_rwmmio_read(CALLER_ADDR0, width, addr); +} +EXPORT_SYMBOL_GPL(__log_read_mmio); +EXPORT_TRACEPOINT_SYMBOL_GPL(rwmmio_read); + +void __log_post_read_mmio(u64 val, u8 width, const volatile void __iomem *addr) +{ + trace_rwmmio_post_read(CALLER_ADDR0, val, width, addr); +} +EXPORT_SYMBOL_GPL(__log_post_read_mmio); +EXPORT_TRACEPOINT_SYMBOL_GPL(rwmmio_post_read); +#endif