From a3775634f6da23f5511d0282d7e792cf606e5f3b Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:21 +0100 Subject: [PATCH 01/67] RISC-V: fix funct4 definition for c.jalr in parse_asm.h The opcode definition for c.jalr is c.jalr c_rs1_n0 1..0=2 15..13=4 12=1 6..2=0 This means funct4 consisting of bit [15:12] is 1001b, so the value is 0x9. Fixes: edde5584c7ab ("riscv: Add SW single-step support for KDB") Reported-by: Andrew Jones Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-2-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/parse_asm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h index f36368de839f..7fee806805c1 100644 --- a/arch/riscv/include/asm/parse_asm.h +++ b/arch/riscv/include/asm/parse_asm.h @@ -125,7 +125,7 @@ #define FUNCT3_C_J 0xa000 #define FUNCT3_C_JAL 0x2000 #define FUNCT4_C_JR 0x8000 -#define FUNCT4_C_JALR 0xf000 +#define FUNCT4_C_JALR 0x9000 #define FUNCT12_SRET 0x10200000 From bf0cc402d7cd4277ece55dd0ace6be43916f713b Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:22 +0100 Subject: [PATCH 02/67] RISC-V: add prefix to all constants/macros in parse_asm.h Some of the constants and macros already have suitable RV_, RVG_ or RVC_ prefixes. Extend this to the rest of the file as well, as we want to use these things in a broader scope soon. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-3-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/parse_asm.h | 188 ++++++++++++++--------------- arch/riscv/kernel/kgdb.c | 40 +++--- 2 files changed, 114 insertions(+), 114 deletions(-) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h index 7fee806805c1..ea51542e0c65 100644 --- a/arch/riscv/include/asm/parse_asm.h +++ b/arch/riscv/include/asm/parse_asm.h @@ -6,37 +6,37 @@ #include /* The bit field of immediate value in I-type instruction */ -#define I_IMM_SIGN_OPOFF 31 -#define I_IMM_11_0_OPOFF 20 -#define I_IMM_SIGN_OFF 12 -#define I_IMM_11_0_OFF 0 -#define I_IMM_11_0_MASK GENMASK(11, 0) +#define RV_I_IMM_SIGN_OPOFF 31 +#define RV_I_IMM_11_0_OPOFF 20 +#define RV_I_IMM_SIGN_OFF 12 +#define RV_I_IMM_11_0_OFF 0 +#define RV_I_IMM_11_0_MASK GENMASK(11, 0) /* The bit field of immediate value in J-type instruction */ -#define J_IMM_SIGN_OPOFF 31 -#define J_IMM_10_1_OPOFF 21 -#define J_IMM_11_OPOFF 20 -#define J_IMM_19_12_OPOFF 12 -#define J_IMM_SIGN_OFF 20 -#define J_IMM_10_1_OFF 1 -#define J_IMM_11_OFF 11 -#define J_IMM_19_12_OFF 12 -#define J_IMM_10_1_MASK GENMASK(9, 0) -#define J_IMM_11_MASK GENMASK(0, 0) -#define J_IMM_19_12_MASK GENMASK(7, 0) +#define RV_J_IMM_SIGN_OPOFF 31 +#define RV_J_IMM_10_1_OPOFF 21 +#define RV_J_IMM_11_OPOFF 20 +#define RV_J_IMM_19_12_OPOFF 12 +#define RV_J_IMM_SIGN_OFF 20 +#define RV_J_IMM_10_1_OFF 1 +#define RV_J_IMM_11_OFF 11 +#define RV_J_IMM_19_12_OFF 12 +#define RV_J_IMM_10_1_MASK GENMASK(9, 0) +#define RV_J_IMM_11_MASK GENMASK(0, 0) +#define RV_J_IMM_19_12_MASK GENMASK(7, 0) /* The bit field of immediate value in B-type instruction */ -#define B_IMM_SIGN_OPOFF 31 -#define B_IMM_10_5_OPOFF 25 -#define B_IMM_4_1_OPOFF 8 -#define B_IMM_11_OPOFF 7 -#define B_IMM_SIGN_OFF 12 -#define B_IMM_10_5_OFF 5 -#define B_IMM_4_1_OFF 1 -#define B_IMM_11_OFF 11 -#define B_IMM_10_5_MASK GENMASK(5, 0) -#define B_IMM_4_1_MASK GENMASK(3, 0) -#define B_IMM_11_MASK GENMASK(0, 0) +#define RV_B_IMM_SIGN_OPOFF 31 +#define RV_B_IMM_10_5_OPOFF 25 +#define RV_B_IMM_4_1_OPOFF 8 +#define RV_B_IMM_11_OPOFF 7 +#define RV_B_IMM_SIGN_OFF 12 +#define RV_B_IMM_10_5_OFF 5 +#define RV_B_IMM_4_1_OFF 1 +#define RV_B_IMM_11_OFF 11 +#define RV_B_IMM_10_5_MASK GENMASK(5, 0) +#define RV_B_IMM_4_1_MASK GENMASK(3, 0) +#define RV_B_IMM_11_MASK GENMASK(0, 0) /* The register offset in RVG instruction */ #define RVG_RS1_OPOFF 15 @@ -100,71 +100,71 @@ #define RVC_C2_RD_OPOFF 7 /* parts of opcode for RVG*/ -#define OPCODE_BRANCH 0x63 -#define OPCODE_JALR 0x67 -#define OPCODE_JAL 0x6f -#define OPCODE_SYSTEM 0x73 +#define RVG_OPCODE_BRANCH 0x63 +#define RVG_OPCODE_JALR 0x67 +#define RVG_OPCODE_JAL 0x6f +#define RVG_OPCODE_SYSTEM 0x73 /* parts of opcode for RVC*/ -#define OPCODE_C_0 0x0 -#define OPCODE_C_1 0x1 -#define OPCODE_C_2 0x2 +#define RVC_OPCODE_C0 0x0 +#define RVC_OPCODE_C1 0x1 +#define RVC_OPCODE_C2 0x2 /* parts of funct3 code for I, M, A extension*/ -#define FUNCT3_JALR 0x0 -#define FUNCT3_BEQ 0x0 -#define FUNCT3_BNE 0x1000 -#define FUNCT3_BLT 0x4000 -#define FUNCT3_BGE 0x5000 -#define FUNCT3_BLTU 0x6000 -#define FUNCT3_BGEU 0x7000 +#define RVG_FUNCT3_JALR 0x0 +#define RVG_FUNCT3_BEQ 0x0 +#define RVG_FUNCT3_BNE 0x1000 +#define RVG_FUNCT3_BLT 0x4000 +#define RVG_FUNCT3_BGE 0x5000 +#define RVG_FUNCT3_BLTU 0x6000 +#define RVG_FUNCT3_BGEU 0x7000 /* parts of funct3 code for C extension*/ -#define FUNCT3_C_BEQZ 0xc000 -#define FUNCT3_C_BNEZ 0xe000 -#define FUNCT3_C_J 0xa000 -#define FUNCT3_C_JAL 0x2000 -#define FUNCT4_C_JR 0x8000 -#define FUNCT4_C_JALR 0x9000 +#define RVC_FUNCT3_C_BEQZ 0xc000 +#define RVC_FUNCT3_C_BNEZ 0xe000 +#define RVC_FUNCT3_C_J 0xa000 +#define RVC_FUNCT3_C_JAL 0x2000 +#define RVC_FUNCT4_C_JR 0x8000 +#define RVC_FUNCT4_C_JALR 0x9000 -#define FUNCT12_SRET 0x10200000 +#define RVG_FUNCT12_SRET 0x10200000 -#define MATCH_JALR (FUNCT3_JALR | OPCODE_JALR) -#define MATCH_JAL (OPCODE_JAL) -#define MATCH_BEQ (FUNCT3_BEQ | OPCODE_BRANCH) -#define MATCH_BNE (FUNCT3_BNE | OPCODE_BRANCH) -#define MATCH_BLT (FUNCT3_BLT | OPCODE_BRANCH) -#define MATCH_BGE (FUNCT3_BGE | OPCODE_BRANCH) -#define MATCH_BLTU (FUNCT3_BLTU | OPCODE_BRANCH) -#define MATCH_BGEU (FUNCT3_BGEU | OPCODE_BRANCH) -#define MATCH_SRET (FUNCT12_SRET | OPCODE_SYSTEM) -#define MATCH_C_BEQZ (FUNCT3_C_BEQZ | OPCODE_C_1) -#define MATCH_C_BNEZ (FUNCT3_C_BNEZ | OPCODE_C_1) -#define MATCH_C_J (FUNCT3_C_J | OPCODE_C_1) -#define MATCH_C_JAL (FUNCT3_C_JAL | OPCODE_C_1) -#define MATCH_C_JR (FUNCT4_C_JR | OPCODE_C_2) -#define MATCH_C_JALR (FUNCT4_C_JALR | OPCODE_C_2) +#define RVG_MATCH_JALR (RVG_FUNCT3_JALR | RVG_OPCODE_JALR) +#define RVG_MATCH_JAL (RVG_OPCODE_JAL) +#define RVG_MATCH_BEQ (RVG_FUNCT3_BEQ | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BNE (RVG_FUNCT3_BNE | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BLT (RVG_FUNCT3_BLT | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BGE (RVG_FUNCT3_BGE | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BLTU (RVG_FUNCT3_BLTU | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BGEU (RVG_FUNCT3_BGEU | RVG_OPCODE_BRANCH) +#define RVG_MATCH_SRET (RVG_FUNCT12_SRET | RVG_OPCODE_SYSTEM) +#define RVC_MATCH_C_BEQZ (RVC_FUNCT3_C_BEQZ | RVC_OPCODE_C1) +#define RVC_MATCH_C_BNEZ (RVC_FUNCT3_C_BNEZ | RVC_OPCODE_C1) +#define RVC_MATCH_C_J (RVC_FUNCT3_C_J | RVC_OPCODE_C1) +#define RVC_MATCH_C_JAL (RVC_FUNCT3_C_JAL | RVC_OPCODE_C1) +#define RVC_MATCH_C_JR (RVC_FUNCT4_C_JR | RVC_OPCODE_C2) +#define RVC_MATCH_C_JALR (RVC_FUNCT4_C_JALR | RVC_OPCODE_C2) -#define MASK_JALR 0x707f -#define MASK_JAL 0x7f -#define MASK_C_JALR 0xf07f -#define MASK_C_JR 0xf07f -#define MASK_C_JAL 0xe003 -#define MASK_C_J 0xe003 -#define MASK_BEQ 0x707f -#define MASK_BNE 0x707f -#define MASK_BLT 0x707f -#define MASK_BGE 0x707f -#define MASK_BLTU 0x707f -#define MASK_BGEU 0x707f -#define MASK_C_BEQZ 0xe003 -#define MASK_C_BNEZ 0xe003 -#define MASK_SRET 0xffffffff +#define RVG_MASK_JALR 0x707f +#define RVG_MASK_JAL 0x7f +#define RVC_MASK_C_JALR 0xf07f +#define RVC_MASK_C_JR 0xf07f +#define RVC_MASK_C_JAL 0xe003 +#define RVC_MASK_C_J 0xe003 +#define RVG_MASK_BEQ 0x707f +#define RVG_MASK_BNE 0x707f +#define RVG_MASK_BLT 0x707f +#define RVG_MASK_BGE 0x707f +#define RVG_MASK_BLTU 0x707f +#define RVG_MASK_BGEU 0x707f +#define RVC_MASK_C_BEQZ 0xe003 +#define RVC_MASK_C_BNEZ 0xe003 +#define RVG_MASK_SRET 0xffffffff #define __INSN_LENGTH_MASK _UL(0x3) #define __INSN_LENGTH_GE_32 _UL(0x3) #define __INSN_OPCODE_MASK _UL(0x7F) -#define __INSN_BRANCH_OPCODE _UL(OPCODE_BRANCH) +#define __INSN_BRANCH_OPCODE _UL(RVG_OPCODE_BRANCH) /* Define a series of is_XXX_insn functions to check if the value INSN * is an instance of instruction XXX. @@ -180,26 +180,26 @@ static inline bool is_ ## INSN_NAME ## _insn(long insn) \ #define RV_X(X, s, mask) (((X) >> (s)) & (mask)) #define RVC_X(X, s, mask) RV_X(X, s, mask) -#define EXTRACT_JTYPE_IMM(x) \ +#define RV_EXTRACT_JTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, J_IMM_10_1_OPOFF, J_IMM_10_1_MASK) << J_IMM_10_1_OFF) | \ - (RV_X(x_, J_IMM_11_OPOFF, J_IMM_11_MASK) << J_IMM_11_OFF) | \ - (RV_X(x_, J_IMM_19_12_OPOFF, J_IMM_19_12_MASK) << J_IMM_19_12_OFF) | \ - (RV_IMM_SIGN(x_) << J_IMM_SIGN_OFF); }) + (RV_X(x_, RV_J_IMM_10_1_OPOFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OFF) | \ + (RV_X(x_, RV_J_IMM_11_OPOFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OFF) | \ + (RV_X(x_, RV_J_IMM_19_12_OPOFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OFF) | \ + (RV_IMM_SIGN(x_) << RV_J_IMM_SIGN_OFF); }) -#define EXTRACT_ITYPE_IMM(x) \ +#define RV_EXTRACT_ITYPE_IMM(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, I_IMM_11_0_OPOFF, I_IMM_11_0_MASK)) | \ - (RV_IMM_SIGN(x_) << I_IMM_SIGN_OFF); }) + (RV_X(x_, RV_I_IMM_11_0_OPOFF, RV_I_IMM_11_0_MASK)) | \ + (RV_IMM_SIGN(x_) << RV_I_IMM_SIGN_OFF); }) -#define EXTRACT_BTYPE_IMM(x) \ +#define RV_EXTRACT_BTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, B_IMM_4_1_OPOFF, B_IMM_4_1_MASK) << B_IMM_4_1_OFF) | \ - (RV_X(x_, B_IMM_10_5_OPOFF, B_IMM_10_5_MASK) << B_IMM_10_5_OFF) | \ - (RV_X(x_, B_IMM_11_OPOFF, B_IMM_11_MASK) << B_IMM_11_OFF) | \ - (RV_IMM_SIGN(x_) << B_IMM_SIGN_OFF); }) + (RV_X(x_, RV_B_IMM_4_1_OPOFF, RV_B_IMM_4_1_MASK) << RV_B_IMM_4_1_OFF) | \ + (RV_X(x_, RV_B_IMM_10_5_OPOFF, RV_B_IMM_10_5_MASK) << RV_B_IMM_10_5_OFF) | \ + (RV_X(x_, RV_B_IMM_11_OPOFF, RV_B_IMM_11_MASK) << RV_B_IMM_11_OFF) | \ + (RV_IMM_SIGN(x_) << RV_B_IMM_SIGN_OFF); }) -#define EXTRACT_RVC_J_IMM(x) \ +#define RVC_EXTRACT_JTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ (RVC_X(x_, RVC_J_IMM_3_1_OPOFF, RVC_J_IMM_3_1_MASK) << RVC_J_IMM_3_1_OFF) | \ (RVC_X(x_, RVC_J_IMM_4_OPOFF, RVC_J_IMM_4_MASK) << RVC_J_IMM_4_OFF) | \ @@ -210,7 +210,7 @@ static inline bool is_ ## INSN_NAME ## _insn(long insn) \ (RVC_X(x_, RVC_J_IMM_10_OPOFF, RVC_J_IMM_10_MASK) << RVC_J_IMM_10_OFF) | \ (RVC_IMM_SIGN(x_) << RVC_J_IMM_SIGN_OFF); }) -#define EXTRACT_RVC_B_IMM(x) \ +#define RVC_EXTRACT_BTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ (RVC_X(x_, RVC_B_IMM_2_1_OPOFF, RVC_B_IMM_2_1_MASK) << RVC_B_IMM_2_1_OFF) | \ (RVC_X(x_, RVC_B_IMM_4_3_OPOFF, RVC_B_IMM_4_3_MASK) << RVC_B_IMM_4_3_OFF) | \ diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 963ed7edcff2..137c6d870d58 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -29,20 +29,20 @@ DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) #else #define is_c_jal_insn(opcode) 0 #endif -DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) -DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) -DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) -DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) -DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) -DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) -DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) -DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) -DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) -DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) -DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) -DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) -DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) -DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) +DECLARE_INSN(jalr, RVG_MATCH_JALR, RVG_MASK_JALR) +DECLARE_INSN(jal, RVG_MATCH_JAL, RVG_MASK_JAL) +DECLARE_INSN(c_jr, RVC_MATCH_C_JR, RVC_MASK_C_JR) +DECLARE_INSN(c_jalr, RVC_MATCH_C_JALR, RVC_MASK_C_JALR) +DECLARE_INSN(c_j, RVC_MATCH_C_J, RVC_MASK_C_J) +DECLARE_INSN(beq, RVG_MATCH_BEQ, RVG_MASK_BEQ) +DECLARE_INSN(bne, RVG_MATCH_BNE, RVG_MASK_BNE) +DECLARE_INSN(blt, RVG_MATCH_BLT, RVG_MASK_BLT) +DECLARE_INSN(bge, RVG_MATCH_BGE, RVG_MASK_BGE) +DECLARE_INSN(bltu, RVG_MATCH_BLTU, RVG_MASK_BLTU) +DECLARE_INSN(bgeu, RVG_MATCH_BGEU, RVG_MASK_BGEU) +DECLARE_INSN(c_beqz, RVC_MATCH_C_BEQZ, RVC_MASK_C_BEQZ) +DECLARE_INSN(c_bnez, RVC_MATCH_C_BNEZ, RVC_MASK_C_BNEZ) +DECLARE_INSN(sret, RVG_MATCH_SRET, RVG_MASK_SRET) static int decode_register_index(unsigned long opcode, int offset) { @@ -69,19 +69,19 @@ static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) rs1_num = decode_register_index(op_code, RVC_C2_RS1_OPOFF); *next_addr = regs_ptr[rs1_num]; } else if (is_c_j_insn(op_code) || is_c_jal_insn(op_code)) { - *next_addr = EXTRACT_RVC_J_IMM(op_code) + pc; + *next_addr = RVC_EXTRACT_JTYPE_IMM(op_code) + pc; } else if (is_c_beqz_insn(op_code)) { rs1_num = decode_register_index_short(op_code, RVC_C1_RS1_OPOFF); if (!rs1_num || regs_ptr[rs1_num] == 0) - *next_addr = EXTRACT_RVC_B_IMM(op_code) + pc; + *next_addr = RVC_EXTRACT_BTYPE_IMM(op_code) + pc; else *next_addr = pc + 2; } else if (is_c_bnez_insn(op_code)) { rs1_num = decode_register_index_short(op_code, RVC_C1_RS1_OPOFF); if (rs1_num && regs_ptr[rs1_num] != 0) - *next_addr = EXTRACT_RVC_B_IMM(op_code) + pc; + *next_addr = RVC_EXTRACT_BTYPE_IMM(op_code) + pc; else *next_addr = pc + 2; } else { @@ -90,7 +90,7 @@ static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) } else { if ((op_code & __INSN_OPCODE_MASK) == __INSN_BRANCH_OPCODE) { bool result = false; - long imm = EXTRACT_BTYPE_IMM(op_code); + long imm = RV_EXTRACT_BTYPE_IMM(op_code); unsigned long rs1_val = 0, rs2_val = 0; rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); @@ -121,12 +121,12 @@ static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) else *next_addr = pc + 4; } else if (is_jal_insn(op_code)) { - *next_addr = EXTRACT_JTYPE_IMM(op_code) + pc; + *next_addr = RV_EXTRACT_JTYPE_IMM(op_code) + pc; } else if (is_jalr_insn(op_code)) { rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); if (rs1_num) *next_addr = ((unsigned long *)regs)[rs1_num]; - *next_addr += EXTRACT_ITYPE_IMM(op_code); + *next_addr += RV_EXTRACT_ITYPE_IMM(op_code); } else if (is_sret_insn(op_code)) { *next_addr = pc; } else { From 33e41480b233eb3ae8e3b523062a1916693ac267 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:23 +0100 Subject: [PATCH 03/67] RISC-V: detach funct-values from their offset Rather than defining funct3, funct4, etc values pre-shifted to their target-position in an instruction, define the values themselves and only shift them where needed. This allows using these funct-values in other places as well, for example when decoding functions. At the same time also reduces the use of magic numbers, one would need a spec manual to understand. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-4-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/parse_asm.h | 98 ++++++++++++++++++------------ 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h index ea51542e0c65..e3f87da108f4 100644 --- a/arch/riscv/include/asm/parse_asm.h +++ b/arch/riscv/include/asm/parse_asm.h @@ -5,6 +5,15 @@ #include +#define RV_INSN_FUNCT3_MASK GENMASK(14, 12) +#define RV_INSN_FUNCT3_OPOFF 12 +#define RV_INSN_OPCODE_MASK GENMASK(6, 0) +#define RV_INSN_OPCODE_OPOFF 0 +#define RV_INSN_FUNCT12_OPOFF 20 + +#define RV_ENCODE_FUNCT3(f_) (RVG_FUNCT3_##f_ << RV_INSN_FUNCT3_OPOFF) +#define RV_ENCODE_FUNCT12(f_) (RVG_FUNCT12_##f_ << RV_INSN_FUNCT12_OPOFF) + /* The bit field of immediate value in I-type instruction */ #define RV_I_IMM_SIGN_OPOFF 31 #define RV_I_IMM_11_0_OPOFF 20 @@ -84,6 +93,15 @@ #define RVC_B_IMM_2_1_MASK GENMASK(1, 0) #define RVC_B_IMM_5_MASK GENMASK(0, 0) +#define RVC_INSN_FUNCT4_MASK GENMASK(15, 12) +#define RVC_INSN_FUNCT4_OPOFF 12 +#define RVC_INSN_FUNCT3_MASK GENMASK(15, 13) +#define RVC_INSN_FUNCT3_OPOFF 13 +#define RVC_INSN_J_RS2_MASK GENMASK(6, 2) +#define RVC_INSN_OPCODE_MASK GENMASK(1, 0) +#define RVC_ENCODE_FUNCT3(f_) (RVC_FUNCT3_##f_ << RVC_INSN_FUNCT3_OPOFF) +#define RVC_ENCODE_FUNCT4(f_) (RVC_FUNCT4_##f_ << RVC_INSN_FUNCT4_OPOFF) + /* The register offset in RVC op=C0 instruction */ #define RVC_C0_RS1_OPOFF 7 #define RVC_C0_RS2_OPOFF 2 @@ -113,52 +131,52 @@ /* parts of funct3 code for I, M, A extension*/ #define RVG_FUNCT3_JALR 0x0 #define RVG_FUNCT3_BEQ 0x0 -#define RVG_FUNCT3_BNE 0x1000 -#define RVG_FUNCT3_BLT 0x4000 -#define RVG_FUNCT3_BGE 0x5000 -#define RVG_FUNCT3_BLTU 0x6000 -#define RVG_FUNCT3_BGEU 0x7000 +#define RVG_FUNCT3_BNE 0x1 +#define RVG_FUNCT3_BLT 0x4 +#define RVG_FUNCT3_BGE 0x5 +#define RVG_FUNCT3_BLTU 0x6 +#define RVG_FUNCT3_BGEU 0x7 /* parts of funct3 code for C extension*/ -#define RVC_FUNCT3_C_BEQZ 0xc000 -#define RVC_FUNCT3_C_BNEZ 0xe000 -#define RVC_FUNCT3_C_J 0xa000 -#define RVC_FUNCT3_C_JAL 0x2000 -#define RVC_FUNCT4_C_JR 0x8000 -#define RVC_FUNCT4_C_JALR 0x9000 +#define RVC_FUNCT3_C_BEQZ 0x6 +#define RVC_FUNCT3_C_BNEZ 0x7 +#define RVC_FUNCT3_C_J 0x5 +#define RVC_FUNCT3_C_JAL 0x1 +#define RVC_FUNCT4_C_JR 0x8 +#define RVC_FUNCT4_C_JALR 0x9 -#define RVG_FUNCT12_SRET 0x10200000 +#define RVG_FUNCT12_SRET 0x102 -#define RVG_MATCH_JALR (RVG_FUNCT3_JALR | RVG_OPCODE_JALR) +#define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR) #define RVG_MATCH_JAL (RVG_OPCODE_JAL) -#define RVG_MATCH_BEQ (RVG_FUNCT3_BEQ | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BNE (RVG_FUNCT3_BNE | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BLT (RVG_FUNCT3_BLT | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BGE (RVG_FUNCT3_BGE | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BLTU (RVG_FUNCT3_BLTU | RVG_OPCODE_BRANCH) -#define RVG_MATCH_BGEU (RVG_FUNCT3_BGEU | RVG_OPCODE_BRANCH) -#define RVG_MATCH_SRET (RVG_FUNCT12_SRET | RVG_OPCODE_SYSTEM) -#define RVC_MATCH_C_BEQZ (RVC_FUNCT3_C_BEQZ | RVC_OPCODE_C1) -#define RVC_MATCH_C_BNEZ (RVC_FUNCT3_C_BNEZ | RVC_OPCODE_C1) -#define RVC_MATCH_C_J (RVC_FUNCT3_C_J | RVC_OPCODE_C1) -#define RVC_MATCH_C_JAL (RVC_FUNCT3_C_JAL | RVC_OPCODE_C1) -#define RVC_MATCH_C_JR (RVC_FUNCT4_C_JR | RVC_OPCODE_C2) -#define RVC_MATCH_C_JALR (RVC_FUNCT4_C_JALR | RVC_OPCODE_C2) +#define RVG_MATCH_BEQ (RV_ENCODE_FUNCT3(BEQ) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BNE (RV_ENCODE_FUNCT3(BNE) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BLT (RV_ENCODE_FUNCT3(BLT) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BGE (RV_ENCODE_FUNCT3(BGE) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BLTU (RV_ENCODE_FUNCT3(BLTU) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BGEU (RV_ENCODE_FUNCT3(BGEU) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_SRET (RV_ENCODE_FUNCT12(SRET) | RVG_OPCODE_SYSTEM) +#define RVC_MATCH_C_BEQZ (RVC_ENCODE_FUNCT3(C_BEQZ) | RVC_OPCODE_C1) +#define RVC_MATCH_C_BNEZ (RVC_ENCODE_FUNCT3(C_BNEZ) | RVC_OPCODE_C1) +#define RVC_MATCH_C_J (RVC_ENCODE_FUNCT3(C_J) | RVC_OPCODE_C1) +#define RVC_MATCH_C_JAL (RVC_ENCODE_FUNCT3(C_JAL) | RVC_OPCODE_C1) +#define RVC_MATCH_C_JR (RVC_ENCODE_FUNCT4(C_JR) | RVC_OPCODE_C2) +#define RVC_MATCH_C_JALR (RVC_ENCODE_FUNCT4(C_JALR) | RVC_OPCODE_C2) -#define RVG_MASK_JALR 0x707f -#define RVG_MASK_JAL 0x7f -#define RVC_MASK_C_JALR 0xf07f -#define RVC_MASK_C_JR 0xf07f -#define RVC_MASK_C_JAL 0xe003 -#define RVC_MASK_C_J 0xe003 -#define RVG_MASK_BEQ 0x707f -#define RVG_MASK_BNE 0x707f -#define RVG_MASK_BLT 0x707f -#define RVG_MASK_BGE 0x707f -#define RVG_MASK_BLTU 0x707f -#define RVG_MASK_BGEU 0x707f -#define RVC_MASK_C_BEQZ 0xe003 -#define RVC_MASK_C_BNEZ 0xe003 +#define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) +#define RVC_MASK_C_JALR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_JR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_JAL (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_J (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVG_MASK_BEQ (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BNE (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BLT (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BGE (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BLTU (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BGEU (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVC_MASK_C_BEQZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_BNEZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) #define RVG_MASK_SRET 0xffffffff #define __INSN_LENGTH_MASK _UL(0x3) From 2621cac76f84c39a0e828f644bb04767efe50981 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:24 +0100 Subject: [PATCH 04/67] RISC-V: add ebreak instructions to definitions kprobes need to match ebreak instructions, so add the necessary data to enable us to centralize that functionality. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-5-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/parse_asm.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h index e3f87da108f4..e8303250f598 100644 --- a/arch/riscv/include/asm/parse_asm.h +++ b/arch/riscv/include/asm/parse_asm.h @@ -144,7 +144,9 @@ #define RVC_FUNCT3_C_JAL 0x1 #define RVC_FUNCT4_C_JR 0x8 #define RVC_FUNCT4_C_JALR 0x9 +#define RVC_FUNCT4_C_EBREAK 0x9 +#define RVG_FUNCT12_EBREAK 0x1 #define RVG_FUNCT12_SRET 0x102 #define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR) @@ -155,6 +157,7 @@ #define RVG_MATCH_BGE (RV_ENCODE_FUNCT3(BGE) | RVG_OPCODE_BRANCH) #define RVG_MATCH_BLTU (RV_ENCODE_FUNCT3(BLTU) | RVG_OPCODE_BRANCH) #define RVG_MATCH_BGEU (RV_ENCODE_FUNCT3(BGEU) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_EBREAK (RV_ENCODE_FUNCT12(EBREAK) | RVG_OPCODE_SYSTEM) #define RVG_MATCH_SRET (RV_ENCODE_FUNCT12(SRET) | RVG_OPCODE_SYSTEM) #define RVC_MATCH_C_BEQZ (RVC_ENCODE_FUNCT3(C_BEQZ) | RVC_OPCODE_C1) #define RVC_MATCH_C_BNEZ (RVC_ENCODE_FUNCT3(C_BNEZ) | RVC_OPCODE_C1) @@ -162,6 +165,7 @@ #define RVC_MATCH_C_JAL (RVC_ENCODE_FUNCT3(C_JAL) | RVC_OPCODE_C1) #define RVC_MATCH_C_JR (RVC_ENCODE_FUNCT4(C_JR) | RVC_OPCODE_C2) #define RVC_MATCH_C_JALR (RVC_ENCODE_FUNCT4(C_JALR) | RVC_OPCODE_C2) +#define RVC_MATCH_C_EBREAK (RVC_ENCODE_FUNCT4(C_EBREAK) | RVC_OPCODE_C2) #define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) #define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) @@ -177,6 +181,8 @@ #define RVG_MASK_BGEU (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) #define RVC_MASK_C_BEQZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) #define RVC_MASK_C_BNEZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_EBREAK 0xffff +#define RVG_MASK_EBREAK 0xffffffff #define RVG_MASK_SRET 0xffffffff #define __INSN_LENGTH_MASK _UL(0x3) From cde4a69302a4a57374a9daa463c493ff45038c99 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:25 +0100 Subject: [PATCH 05/67] RISC-V: add auipc elements to parse_asm header Instruction parsing should not be done in individual code, but instead supported by central Right now kgdb and kprobes parse instructions and at least kprobes (and the upcoming auipc+jalr alternative fixer-function) need the auipc instruction. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-6-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/parse_asm.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h index e8303250f598..28742eb19034 100644 --- a/arch/riscv/include/asm/parse_asm.h +++ b/arch/riscv/include/asm/parse_asm.h @@ -118,6 +118,7 @@ #define RVC_C2_RD_OPOFF 7 /* parts of opcode for RVG*/ +#define RVG_OPCODE_AUIPC 0x17 #define RVG_OPCODE_BRANCH 0x63 #define RVG_OPCODE_JALR 0x67 #define RVG_OPCODE_JAL 0x6f @@ -149,6 +150,7 @@ #define RVG_FUNCT12_EBREAK 0x1 #define RVG_FUNCT12_SRET 0x102 +#define RVG_MATCH_AUIPC (RVG_OPCODE_AUIPC) #define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR) #define RVG_MATCH_JAL (RVG_OPCODE_JAL) #define RVG_MATCH_BEQ (RV_ENCODE_FUNCT3(BEQ) | RVG_OPCODE_BRANCH) @@ -167,6 +169,7 @@ #define RVC_MATCH_C_JALR (RVC_ENCODE_FUNCT4(C_JALR) | RVC_OPCODE_C2) #define RVC_MATCH_C_EBREAK (RVC_ENCODE_FUNCT4(C_EBREAK) | RVC_OPCODE_C2) +#define RVG_MASK_AUIPC (RV_INSN_OPCODE_MASK) #define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) #define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) #define RVC_MASK_C_JALR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) From ec5f908775160d4d635f99cbad9a5ea138894038 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:26 +0100 Subject: [PATCH 06/67] RISC-V: Move riscv_insn_is_* macros into a common header Right now the riscv kernel has (at least) two independent sets of functions to check if an encoded instruction is of a specific type. One in kgdb and one kprobes simulate-insn code. More parts of the kernel will probably need this in the future, so instead of allowing this duplication to go on further, move macros that do the function declaration in a common header, similar to at least aarch64. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-7-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/parse_asm.h | 42 ++++++++++++++++---- arch/riscv/kernel/kgdb.c | 49 ++++++++---------------- arch/riscv/kernel/probes/simulate-insn.h | 26 +++---------- 3 files changed, 55 insertions(+), 62 deletions(-) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/parse_asm.h index 28742eb19034..50c899cf4de5 100644 --- a/arch/riscv/include/asm/parse_asm.h +++ b/arch/riscv/include/asm/parse_asm.h @@ -193,13 +193,41 @@ #define __INSN_OPCODE_MASK _UL(0x7F) #define __INSN_BRANCH_OPCODE _UL(RVG_OPCODE_BRANCH) -/* Define a series of is_XXX_insn functions to check if the value INSN - * is an instance of instruction XXX. - */ -#define DECLARE_INSN(INSN_NAME, INSN_MATCH, INSN_MASK) \ -static inline bool is_ ## INSN_NAME ## _insn(long insn) \ -{ \ - return (insn & (INSN_MASK)) == (INSN_MATCH); \ +#define __RISCV_INSN_FUNCS(name, mask, val) \ +static __always_inline bool riscv_insn_is_##name(u32 code) \ +{ \ + BUILD_BUG_ON(~(mask) & (val)); \ + return (code & (mask)) == (val); \ +} \ + +#if __riscv_xlen == 32 +/* C.JAL is an RV32C-only instruction */ +__RISCV_INSN_FUNCS(c_jal, RVC_MASK_C_JAL, RVC_MATCH_C_JAL) +#else +#define riscv_insn_is_c_jal(opcode) 0 +#endif +__RISCV_INSN_FUNCS(auipc, RVG_MASK_AUIPC, RVG_MATCH_AUIPC) +__RISCV_INSN_FUNCS(jalr, RVG_MASK_JALR, RVG_MATCH_JALR) +__RISCV_INSN_FUNCS(jal, RVG_MASK_JAL, RVG_MATCH_JAL) +__RISCV_INSN_FUNCS(c_jr, RVC_MASK_C_JR, RVC_MATCH_C_JR) +__RISCV_INSN_FUNCS(c_jalr, RVC_MASK_C_JALR, RVC_MATCH_C_JALR) +__RISCV_INSN_FUNCS(c_j, RVC_MASK_C_J, RVC_MATCH_C_J) +__RISCV_INSN_FUNCS(beq, RVG_MASK_BEQ, RVG_MATCH_BEQ) +__RISCV_INSN_FUNCS(bne, RVG_MASK_BNE, RVG_MATCH_BNE) +__RISCV_INSN_FUNCS(blt, RVG_MASK_BLT, RVG_MATCH_BLT) +__RISCV_INSN_FUNCS(bge, RVG_MASK_BGE, RVG_MATCH_BGE) +__RISCV_INSN_FUNCS(bltu, RVG_MASK_BLTU, RVG_MATCH_BLTU) +__RISCV_INSN_FUNCS(bgeu, RVG_MASK_BGEU, RVG_MATCH_BGEU) +__RISCV_INSN_FUNCS(c_beqz, RVC_MASK_C_BEQZ, RVC_MATCH_C_BEQZ) +__RISCV_INSN_FUNCS(c_bnez, RVC_MASK_C_BNEZ, RVC_MATCH_C_BNEZ) +__RISCV_INSN_FUNCS(c_ebreak, RVC_MASK_C_EBREAK, RVC_MATCH_C_EBREAK) +__RISCV_INSN_FUNCS(ebreak, RVG_MASK_EBREAK, RVG_MATCH_EBREAK) +__RISCV_INSN_FUNCS(sret, RVG_MASK_SRET, RVG_MATCH_SRET) + +/* special case to catch _any_ branch instruction */ +static __always_inline bool riscv_insn_is_branch(u32 code) +{ + return (code & RV_INSN_OPCODE_MASK) == RVG_OPCODE_BRANCH; } #define RV_IMM_SIGN(x) (-(((x) >> 31) & 1)) diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 137c6d870d58..61237aeb493c 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -23,27 +23,6 @@ enum { static unsigned long stepped_address; static unsigned int stepped_opcode; -#if __riscv_xlen == 32 -/* C.JAL is an RV32C-only instruction */ -DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) -#else -#define is_c_jal_insn(opcode) 0 -#endif -DECLARE_INSN(jalr, RVG_MATCH_JALR, RVG_MASK_JALR) -DECLARE_INSN(jal, RVG_MATCH_JAL, RVG_MASK_JAL) -DECLARE_INSN(c_jr, RVC_MATCH_C_JR, RVC_MASK_C_JR) -DECLARE_INSN(c_jalr, RVC_MATCH_C_JALR, RVC_MASK_C_JALR) -DECLARE_INSN(c_j, RVC_MATCH_C_J, RVC_MASK_C_J) -DECLARE_INSN(beq, RVG_MATCH_BEQ, RVG_MASK_BEQ) -DECLARE_INSN(bne, RVG_MATCH_BNE, RVG_MASK_BNE) -DECLARE_INSN(blt, RVG_MATCH_BLT, RVG_MASK_BLT) -DECLARE_INSN(bge, RVG_MATCH_BGE, RVG_MASK_BGE) -DECLARE_INSN(bltu, RVG_MATCH_BLTU, RVG_MASK_BLTU) -DECLARE_INSN(bgeu, RVG_MATCH_BGEU, RVG_MASK_BGEU) -DECLARE_INSN(c_beqz, RVC_MATCH_C_BEQZ, RVC_MASK_C_BEQZ) -DECLARE_INSN(c_bnez, RVC_MATCH_C_BNEZ, RVC_MASK_C_BNEZ) -DECLARE_INSN(sret, RVG_MATCH_SRET, RVG_MASK_SRET) - static int decode_register_index(unsigned long opcode, int offset) { return (opcode >> offset) & 0x1F; @@ -65,19 +44,21 @@ static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) if (get_kernel_nofault(op_code, (void *)pc)) return -EINVAL; if ((op_code & __INSN_LENGTH_MASK) != __INSN_LENGTH_GE_32) { - if (is_c_jalr_insn(op_code) || is_c_jr_insn(op_code)) { + if (riscv_insn_is_c_jalr(op_code) || + riscv_insn_is_c_jr(op_code)) { rs1_num = decode_register_index(op_code, RVC_C2_RS1_OPOFF); *next_addr = regs_ptr[rs1_num]; - } else if (is_c_j_insn(op_code) || is_c_jal_insn(op_code)) { + } else if (riscv_insn_is_c_j(op_code) || + riscv_insn_is_c_jal(op_code)) { *next_addr = RVC_EXTRACT_JTYPE_IMM(op_code) + pc; - } else if (is_c_beqz_insn(op_code)) { + } else if (riscv_insn_is_c_beqz(op_code)) { rs1_num = decode_register_index_short(op_code, RVC_C1_RS1_OPOFF); if (!rs1_num || regs_ptr[rs1_num] == 0) *next_addr = RVC_EXTRACT_BTYPE_IMM(op_code) + pc; else *next_addr = pc + 2; - } else if (is_c_bnez_insn(op_code)) { + } else if (riscv_insn_is_c_bnez(op_code)) { rs1_num = decode_register_index_short(op_code, RVC_C1_RS1_OPOFF); if (rs1_num && regs_ptr[rs1_num] != 0) @@ -100,34 +81,34 @@ static int get_step_address(struct pt_regs *regs, unsigned long *next_addr) if (rs2_num) rs2_val = regs_ptr[rs2_num]; - if (is_beq_insn(op_code)) + if (riscv_insn_is_beq(op_code)) result = (rs1_val == rs2_val) ? true : false; - else if (is_bne_insn(op_code)) + else if (riscv_insn_is_bne(op_code)) result = (rs1_val != rs2_val) ? true : false; - else if (is_blt_insn(op_code)) + else if (riscv_insn_is_blt(op_code)) result = ((long)rs1_val < (long)rs2_val) ? true : false; - else if (is_bge_insn(op_code)) + else if (riscv_insn_is_bge(op_code)) result = ((long)rs1_val >= (long)rs2_val) ? true : false; - else if (is_bltu_insn(op_code)) + else if (riscv_insn_is_bltu(op_code)) result = (rs1_val < rs2_val) ? true : false; - else if (is_bgeu_insn(op_code)) + else if (riscv_insn_is_bgeu(op_code)) result = (rs1_val >= rs2_val) ? true : false; if (result) *next_addr = imm + pc; else *next_addr = pc + 4; - } else if (is_jal_insn(op_code)) { + } else if (riscv_insn_is_jal(op_code)) { *next_addr = RV_EXTRACT_JTYPE_IMM(op_code) + pc; - } else if (is_jalr_insn(op_code)) { + } else if (riscv_insn_is_jalr(op_code)) { rs1_num = decode_register_index(op_code, RVG_RS1_OPOFF); if (rs1_num) *next_addr = ((unsigned long *)regs)[rs1_num]; *next_addr += RV_EXTRACT_ITYPE_IMM(op_code); - } else if (is_sret_insn(op_code)) { + } else if (riscv_insn_is_sret(op_code)) { *next_addr = pc; } else { *next_addr = pc + 4; diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h index cb6ff7dccb92..29fb16cd335c 100644 --- a/arch/riscv/kernel/probes/simulate-insn.h +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -3,14 +3,7 @@ #ifndef _RISCV_KERNEL_PROBES_SIMULATE_INSN_H #define _RISCV_KERNEL_PROBES_SIMULATE_INSN_H -#define __RISCV_INSN_FUNCS(name, mask, val) \ -static __always_inline bool riscv_insn_is_##name(probe_opcode_t code) \ -{ \ - BUILD_BUG_ON(~(mask) & (val)); \ - return (code & (mask)) == (val); \ -} \ -bool simulate_##name(u32 opcode, unsigned long addr, \ - struct pt_regs *regs) +#include #define RISCV_INSN_REJECTED(name, code) \ do { \ @@ -30,18 +23,9 @@ __RISCV_INSN_FUNCS(fence, 0x7f, 0x0f); } \ } while (0) -__RISCV_INSN_FUNCS(c_j, 0xe003, 0xa001); -__RISCV_INSN_FUNCS(c_jr, 0xf007, 0x8002); -__RISCV_INSN_FUNCS(c_jal, 0xe003, 0x2001); -__RISCV_INSN_FUNCS(c_jalr, 0xf007, 0x9002); -__RISCV_INSN_FUNCS(c_beqz, 0xe003, 0xc001); -__RISCV_INSN_FUNCS(c_bnez, 0xe003, 0xe001); -__RISCV_INSN_FUNCS(c_ebreak, 0xffff, 0x9002); - -__RISCV_INSN_FUNCS(auipc, 0x7f, 0x17); -__RISCV_INSN_FUNCS(branch, 0x7f, 0x63); - -__RISCV_INSN_FUNCS(jal, 0x7f, 0x6f); -__RISCV_INSN_FUNCS(jalr, 0x707f, 0x67); +bool simulate_auipc(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs); +bool simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs); #endif /* _RISCV_KERNEL_PROBES_SIMULATE_INSN_H */ From c9c1af3f186a2bdb828792658bb32b518edba692 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:27 +0100 Subject: [PATCH 07/67] RISC-V: rename parse_asm.h to insn.h The current parse_asm header should become a more centralized place for everything concerning parsing and constructing instructions. We already have a header insn-def.h similar to aarch64, so rename parse_asm.h to insn.h (again similar to aarch64) to show that it's meant for more than simple instruction parsing. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-8-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/{parse_asm.h => insn.h} | 0 arch/riscv/kernel/kgdb.c | 2 +- arch/riscv/kernel/probes/simulate-insn.h | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename arch/riscv/include/asm/{parse_asm.h => insn.h} (100%) diff --git a/arch/riscv/include/asm/parse_asm.h b/arch/riscv/include/asm/insn.h similarity index 100% rename from arch/riscv/include/asm/parse_asm.h rename to arch/riscv/include/asm/insn.h diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 61237aeb493c..2e0266ae6bd7 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include enum { NOT_KGDB_BREAK = 0, diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h index 29fb16cd335c..a19aaa0feb44 100644 --- a/arch/riscv/kernel/probes/simulate-insn.h +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -3,7 +3,7 @@ #ifndef _RISCV_KERNEL_PROBES_SIMULATE_INSN_H #define _RISCV_KERNEL_PROBES_SIMULATE_INSN_H -#include +#include #define RISCV_INSN_REJECTED(name, code) \ do { \ From debe28d0a52e972ab4d735ea2e12c2f88ec19d7c Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:28 +0100 Subject: [PATCH 08/67] RISC-V: kprobes: use central defined funct3 constants Don't redefine values that are already available in the central header asm/insn.h . Use the values from there instead. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-9-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/probes/simulate-insn.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c index d73e96f6ed7c..330afe9331a8 100644 --- a/arch/riscv/kernel/probes/simulate-insn.c +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -136,13 +136,6 @@ bool __kprobes simulate_auipc(u32 opcode, unsigned long addr, struct pt_regs *re #define branch_offset(opcode) \ sign_extend32((branch_imm(opcode)), 12) -#define BRANCH_BEQ 0x0 -#define BRANCH_BNE 0x1 -#define BRANCH_BLT 0x4 -#define BRANCH_BGE 0x5 -#define BRANCH_BLTU 0x6 -#define BRANCH_BGEU 0x7 - bool __kprobes simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs) { /* @@ -169,22 +162,22 @@ bool __kprobes simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *r offset_tmp = branch_offset(opcode); switch (branch_funct3(opcode)) { - case BRANCH_BEQ: + case RVG_FUNCT3_BEQ: offset = (rs1_val == rs2_val) ? offset_tmp : 4; break; - case BRANCH_BNE: + case RVG_FUNCT3_BNE: offset = (rs1_val != rs2_val) ? offset_tmp : 4; break; - case BRANCH_BLT: + case RVG_FUNCT3_BLT: offset = ((long)rs1_val < (long)rs2_val) ? offset_tmp : 4; break; - case BRANCH_BGE: + case RVG_FUNCT3_BGE: offset = ((long)rs1_val >= (long)rs2_val) ? offset_tmp : 4; break; - case BRANCH_BLTU: + case RVG_FUNCT3_BLTU: offset = (rs1_val < rs2_val) ? offset_tmp : 4; break; - case BRANCH_BGEU: + case RVG_FUNCT3_BGEU: offset = (rs1_val >= rs2_val) ? offset_tmp : 4; break; default: From 8963d6224009d7ac0523a05ceb9ea5d04ad5d12d Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:29 +0100 Subject: [PATCH 09/67] RISC-V: add U-type imm parsing to insn.h header Similar to other existing types, allow extracting the immediate for a U-type instruction. U-type immediates are special in that regard, that the value in the instruction in bits [31:12] already represents the same bits of the immediate, so no shifting is required. U-type immediates are for example used in the auipc instruction, so these constants make it easier to parse such instructions. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-10-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/insn.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 50c899cf4de5..21ec817abec1 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -34,6 +34,15 @@ #define RV_J_IMM_11_MASK GENMASK(0, 0) #define RV_J_IMM_19_12_MASK GENMASK(7, 0) +/* + * U-type IMMs contain the upper 20bits [31:20] of an immediate with + * the rest filled in by zeros, so no shifting required. Similarly, + * bit31 contains the signed state, so no sign extension necessary. + */ +#define RV_U_IMM_SIGN_OPOFF 31 +#define RV_U_IMM_31_12_OPOFF 0 +#define RV_U_IMM_31_12_MASK GENMASK(31, 12) + /* The bit field of immediate value in B-type instruction */ #define RV_B_IMM_SIGN_OPOFF 31 #define RV_B_IMM_10_5_OPOFF 25 @@ -235,6 +244,10 @@ static __always_inline bool riscv_insn_is_branch(u32 code) #define RV_X(X, s, mask) (((X) >> (s)) & (mask)) #define RVC_X(X, s, mask) RV_X(X, s, mask) +#define RV_EXTRACT_UTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, RV_U_IMM_31_12_OPOFF, RV_U_IMM_31_12_MASK)); }) + #define RV_EXTRACT_JTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ (RV_X(x_, RV_J_IMM_10_1_OPOFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OFF) | \ From 088ec01fce4eaba17ada8b1831be0c172c941657 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:30 +0100 Subject: [PATCH 10/67] RISC-V: add rd reg parsing to insn.h header Add a macro to allow parsing of the rd register from an instruction. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-11-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/insn.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 21ec817abec1..2a23890b4577 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -60,6 +60,7 @@ #define RVG_RS1_OPOFF 15 #define RVG_RS2_OPOFF 20 #define RVG_RD_OPOFF 7 +#define RVG_RD_MASK GENMASK(4, 0) /* The bit field of immediate value in RVC J instruction */ #define RVC_J_IMM_SIGN_OPOFF 12 @@ -244,6 +245,10 @@ static __always_inline bool riscv_insn_is_branch(u32 code) #define RV_X(X, s, mask) (((X) >> (s)) & (mask)) #define RVC_X(X, s, mask) RV_X(X, s, mask) +#define RV_EXTRACT_RD_REG(x) \ + ({typeof(x) x_ = (x); \ + (RV_X(x_, RVG_RD_OPOFF, RVG_RD_MASK)); }) + #define RV_EXTRACT_UTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ (RV_X(x_, RV_U_IMM_31_12_OPOFF, RV_U_IMM_31_12_MASK)); }) From 47f05757d3d898b4756d6d5c06e77a37337823e9 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:31 +0100 Subject: [PATCH 11/67] RISC-V: add helpers for handling immediates in U-type and I-type pairs Used together U-type and I-type instructions can for example be used to generate bigger jumps (i.e. in auipc+jalr pairs) by splitting the value into an upper immediate (i.e. auipc) and a 12bit immediate (i.e. jalr). Due to both immediates being considered signed this creates some corner cases, so add some helper to prevent this from getting duplicated in different places. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-12-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/insn.h | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 2a23890b4577..98453535324a 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -290,3 +290,51 @@ static __always_inline bool riscv_insn_is_branch(u32 code) (RVC_X(x_, RVC_B_IMM_5_OPOFF, RVC_B_IMM_5_MASK) << RVC_B_IMM_5_OFF) | \ (RVC_X(x_, RVC_B_IMM_7_6_OPOFF, RVC_B_IMM_7_6_MASK) << RVC_B_IMM_7_6_OFF) | \ (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) + +/* + * Put together one immediate from a U-type and I-type instruction pair. + * + * The U-type contains an upper immediate, meaning bits[31:12] with [11:0] + * being zero, while the I-type contains a 12bit immediate. + * Combined these can encode larger 32bit values and are used for example + * in auipc + jalr pairs to allow larger jumps. + * + * @utype_insn: instruction containing the upper immediate + * @itype_insn: instruction + * Return: combined immediate + */ +static inline s32 riscv_insn_extract_utype_itype_imm(u32 utype_insn, u32 itype_insn) +{ + s32 imm; + + imm = RV_EXTRACT_UTYPE_IMM(utype_insn); + imm += RV_EXTRACT_ITYPE_IMM(itype_insn); + + return imm; +} + +/* + * Update a set of two instructions (U-type + I-type) with an immediate value. + * + * Used for example in auipc+jalrs pairs the U-type instructions contains + * a 20bit upper immediate representing bits[31:12], while the I-type + * instruction contains a 12bit immediate representing bits[11:0]. + * + * This also takes into account that both separate immediates are + * considered as signed values, so if the I-type immediate becomes + * negative (BIT(11) set) the U-type part gets adjusted. + * + * @utype_insn: pointer to the utype instruction of the pair + * @itype_insn: pointer to the itype instruction of the pair + * @imm: the immediate to insert into the two instructions + */ +static inline void riscv_insn_insert_utype_itype_imm(u32 *utype_insn, u32 *itype_insn, s32 imm) +{ + /* drop possible old IMM values */ + *utype_insn &= ~(RV_U_IMM_31_12_MASK); + *itype_insn &= ~(RV_I_IMM_11_0_MASK << RV_I_IMM_11_0_OPOFF); + + /* add the adapted IMMs */ + *utype_insn |= (imm & RV_U_IMM_31_12_MASK) + ((imm & BIT(11)) << 1); + *itype_insn |= ((imm & RV_I_IMM_11_0_MASK) << RV_I_IMM_11_0_OPOFF); +} From 27c653c06505f084bcb57f7575916d60efb32279 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 23 Dec 2022 23:13:32 +0100 Subject: [PATCH 12/67] RISC-V: fix auipc-jalr addresses in patched alternatives Alternatives live in a different section, so addresses used by call functions will point to wrong locations after the patch got applied. Similar to arm64, adjust the location to consider that offset. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Lad Prabhakar Tested-by: Lad Prabhakar Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221223221332.4127602-13-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/alternative.h | 3 ++ arch/riscv/kernel/alternative.c | 56 ++++++++++++++++++++++++++++ arch/riscv/kernel/cpufeature.c | 5 ++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h index 6511dd73e812..1bd4027d34ca 100644 --- a/arch/riscv/include/asm/alternative.h +++ b/arch/riscv/include/asm/alternative.h @@ -27,6 +27,9 @@ void __init apply_boot_alternatives(void); void __init apply_early_boot_alternatives(void); void apply_module_alternatives(void *start, size_t length); +void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, + int patch_offset); + struct alt_entry { void *old_ptr; /* address of original instruciton or data */ void *alt_ptr; /* address of replacement instruction or data */ diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index a7d26a00beea..6212ea0eed72 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include struct cpu_manufacturer_info_t { unsigned long vendor_id; @@ -53,6 +55,60 @@ static void __init_or_module riscv_fill_cpu_mfr_info(struct cpu_manufacturer_inf } } +static u32 riscv_instruction_at(void *p) +{ + u16 *parcel = p; + + return (u32)parcel[0] | (u32)parcel[1] << 16; +} + +static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, + u32 jalr_insn, int patch_offset) +{ + u32 call[2] = { auipc_insn, jalr_insn }; + s32 imm; + + /* get and adjust new target address */ + imm = riscv_insn_extract_utype_itype_imm(auipc_insn, jalr_insn); + imm -= patch_offset; + + /* update instructions */ + riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm); + + /* patch the call place again */ + patch_text_nosync(ptr, call, sizeof(u32) * 2); +} + +void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, + int patch_offset) +{ + int num_insn = len / sizeof(u32); + int i; + + for (i = 0; i < num_insn; i++) { + u32 insn = riscv_instruction_at(alt_ptr + i * sizeof(u32)); + + /* + * May be the start of an auipc + jalr pair + * Needs to check that at least one more instruction + * is in the list. + */ + if (riscv_insn_is_auipc(insn) && i < num_insn - 1) { + u32 insn2 = riscv_instruction_at(alt_ptr + (i + 1) * sizeof(u32)); + + if (!riscv_insn_is_jalr(insn2)) + continue; + + /* if instruction pair is a call, it will use the ra register */ + if (RV_EXTRACT_RD_REG(insn) != 1) + continue; + + riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), + insn, insn2, patch_offset); + } + } +} + /* * This is called very early in the boot process (directly after we run * a feature detect on the boot CPU). No need to worry about other CPUs diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 93e45560af30..205bbd6b1fce 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -339,8 +339,11 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, } tmp = (1U << alt->errata_id); - if (cpu_req_feature & tmp) + if (cpu_req_feature & tmp) { patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); + riscv_alternative_fix_offsets(alt->old_ptr, alt->alt_len, + alt->old_ptr - alt->alt_ptr); + } } } #endif From 6fb4c593982a01c8a356d1ba3b54dc1852ba1404 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 11 Jan 2023 10:48:49 +0000 Subject: [PATCH 13/67] RISC-V: fix incorrect type of ARCH_CANAAN_K210_DTB_SOURCE When adding the ARCH_ symbol for the builtin K210 dtb, I erroneously used the bool type for something that is quite obviously a string. Unfortunately, there is no such thing as "def_string", but in this case we can use "default" to propagate the value of SOC_CANAAN_K210_DTB_SOURCE to ARCH_CANAAN_K210_DTB_SOURCE. ARCH_CANAAN_K210_DTB_SOURCE is not user selectable, so using olddefconfig etc will update ARCH_CANAAN_K210_DTB_SOURCE to reflect any changes made to SOC_CANAAN_K210_DTB_SOURCE. Fixes: fc43211939bb ("RISC-V: kconfig.socs: convert usage of SOC_CANAAN to ARCH_CANAAN") Reported-by: Geert Uytterhoeven Signed-off-by: Conor Dooley Tested-by: Geert Uytterhoeven Link: https://lore.kernel.org/all/20230111104848.2088516-1-conor.dooley@microchip.com/ Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig.socs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index ce920f627f6d..fb34ae19d454 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -87,7 +87,8 @@ config SOC_CANAAN_K210_DTB_BUILTIN If unsure, say Y. config ARCH_CANAAN_K210_DTB_SOURCE - def_bool SOC_CANAAN_K210_DTB_SOURCE + string + default SOC_CANAAN_K210_DTB_SOURCE config SOC_CANAAN_K210_DTB_SOURCE string "Source file for the Canaan Kendryte K210 builtin DTB" From 99e2266f2460e5778560f81982b6301dd2a16502 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Mon, 5 Dec 2022 14:45:24 +0000 Subject: [PATCH 14/67] RISC-V: clarify ISA string ordering rules in cpu.c While the current list of rules may have been accurate when created it now lacks some clarity in the face of isa-manual updates. Instead of trying to continuously align this rule-set with the one in the specifications, change the role of this comment. This particular comment is important, as the array it "decorates" defines the order in which the ISA string appears to userspace in /proc/cpuinfo. Re-jig and strengthen the wording to provide contributors with a set order in which to add entries & note why this particular struct needs more attention than others. While in the area, add some whitespace and tweak some wording for readability's sake. Suggested-by: Andrew Jones Reviewed-by: Andrew Jones Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20221205144525.2148448-2-conor.dooley@microchip.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpu.c | 49 ++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index fa427bdcf773..4480c2833ecc 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -120,22 +120,45 @@ device_initcall(riscv_cpuinfo_init); .uprop = #UPROP, \ .isa_ext_id = EXTID, \ } + /* - * Here are the ordering rules of extension naming defined by RISC-V - * specification : - * 1. All extensions should be separated from other multi-letter extensions - * by an underscore. - * 2. The first letter following the 'Z' conventionally indicates the most + * The canonical order of ISA extension names in the ISA string is defined in + * chapter 27 of the unprivileged specification. + * + * Ordinarily, for in-kernel data structures, this order is unimportant but + * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. + * + * The specification uses vague wording, such as should, when it comes to + * ordering, so for our purposes the following rules apply: + * + * 1. All multi-letter extensions must be separated from other extensions by an + * underscore. + * + * 2. Additional standard extensions (starting with 'Z') must be sorted after + * single-letter extensions and before any higher-privileged extensions. + + * 3. The first letter following the 'Z' conventionally indicates the most * closely related alphabetical extension category, IMAFDQLCBKJTPVH. - * If multiple 'Z' extensions are named, they should be ordered first - * by category, then alphabetically within a category. - * 3. Standard supervisor-level extensions (starts with 'S') should be - * listed after standard unprivileged extensions. If multiple - * supervisor-level extensions are listed, they should be ordered + * If multiple 'Z' extensions are named, they must be ordered first by + * category, then alphabetically within a category. + * + * 3. Standard supervisor-level extensions (starting with 'S') must be listed + * after standard unprivileged extensions. If multiple supervisor-level + * extensions are listed, they must be ordered alphabetically. + * + * 4. Standard machine-level extensions (starting with 'Zxm') must be listed + * after any lower-privileged, standard extensions. If multiple + * machine-level extensions are listed, they must be ordered * alphabetically. - * 4. Non-standard extensions (starts with 'X') must be listed after all - * standard extensions. They must be separated from other multi-letter - * extensions by an underscore. + * + * 5. Non-standard extensions (starting with 'X') must be listed after all + * standard extensions. If multiple non-standard extensions are listed, they + * must be ordered alphabetically. + * + * An example string following the order is: + * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux + * + * New entries to this struct should follow the ordering rules described above. */ static struct riscv_isa_ext_data isa_ext_arr[] = { __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), From 80c200b34ee8a0a3378d2073bd8eaae09651c60e Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Mon, 5 Dec 2022 14:45:25 +0000 Subject: [PATCH 15/67] RISC-V: resort all extensions in consistent orders Ordering between each and every list of extensions is wildly inconsistent. Per discussion on the lists pick the following policy: - The array defining order in /proc/cpuinfo follows a narrow interpretation of the ISA specifications, described in a comment immediately presiding it. - All other lists of extensions are sorted alphabetically. This will hopefully allow for easier review & future additions, and reduce conflicts between patchsets as the number of extensions grows. Link: https://lore.kernel.org/all/20221129144742.2935581-2-conor.dooley@microchip.com/ Suggested-by: Andrew Jones Reviewed-by: Andrew Jones Reviewed-by: Heiko Stuebner Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20221205144525.2148448-3-conor.dooley@microchip.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 12 +++++++----- arch/riscv/kernel/cpu.c | 4 ++-- arch/riscv/kernel/cpufeature.c | 6 ++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index b22525290073..ce522aad641a 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -51,14 +51,15 @@ extern unsigned long elf_hwcap; * RISCV_ISA_EXT_MAX. 0-25 range is reserved for single letter * extensions while all the multi-letter extensions should define the next * available logical extension id. + * Entries are sorted alphabetically. */ enum riscv_isa_ext_id { RISCV_ISA_EXT_SSCOFPMF = RISCV_ISA_EXT_BASE, + RISCV_ISA_EXT_SSTC, + RISCV_ISA_EXT_SVINVAL, RISCV_ISA_EXT_SVPBMT, RISCV_ISA_EXT_ZICBOM, RISCV_ISA_EXT_ZIHINTPAUSE, - RISCV_ISA_EXT_SSTC, - RISCV_ISA_EXT_SVINVAL, RISCV_ISA_EXT_ID_MAX = RISCV_ISA_EXT_MAX, }; @@ -66,11 +67,12 @@ enum riscv_isa_ext_id { * This enum represents the logical ID for each RISC-V ISA extension static * keys. We can use static key to optimize code path if some ISA extensions * are available. + * Entries are sorted alphabetically. */ enum riscv_isa_ext_key { RISCV_ISA_EXT_KEY_FPU, /* For 'F' and 'D' */ - RISCV_ISA_EXT_KEY_ZIHINTPAUSE, RISCV_ISA_EXT_KEY_SVINVAL, + RISCV_ISA_EXT_KEY_ZIHINTPAUSE, RISCV_ISA_EXT_KEY_MAX, }; @@ -90,10 +92,10 @@ static __always_inline int riscv_isa_ext2key(int num) return RISCV_ISA_EXT_KEY_FPU; case RISCV_ISA_EXT_d: return RISCV_ISA_EXT_KEY_FPU; - case RISCV_ISA_EXT_ZIHINTPAUSE: - return RISCV_ISA_EXT_KEY_ZIHINTPAUSE; case RISCV_ISA_EXT_SVINVAL: return RISCV_ISA_EXT_KEY_SVINVAL; + case RISCV_ISA_EXT_ZIHINTPAUSE: + return RISCV_ISA_EXT_KEY_ZIHINTPAUSE; default: return -EINVAL; } diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 4480c2833ecc..b8127bfc8f0f 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -161,12 +161,12 @@ device_initcall(riscv_cpuinfo_init); * New entries to this struct should follow the ordering rules described above. */ static struct riscv_isa_ext_data isa_ext_arr[] = { + __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), + __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), - __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), - __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA("", RISCV_ISA_EXT_MAX), }; diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 694267d1fe81..8a76a6ce70cf 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -199,12 +199,13 @@ void __init riscv_fill_hwcap(void) this_hwcap |= isa2hwcap[(unsigned char)(*ext)]; set_bit(*ext - 'a', this_isa); } else { + /* sorted alphabetically */ SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); + SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); + SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); - SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); - SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); } #undef SET_ISA_EXT_MAP } @@ -284,6 +285,7 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) * This code may also be executed before kernel relocation, so we cannot use * addresses generated by the address-of operator as they won't be valid in * this context. + * Tests, unless otherwise required, are to be added in alphabetical order. */ static u32 __init_or_module cpufeature_probe(unsigned int stage) { From f07b2b3f9d47fea308af3ae05613b6b4801e68a3 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Mon, 5 Dec 2022 14:45:26 +0000 Subject: [PATCH 16/67] Documentation: riscv: add a section about ISA string ordering in /proc/cpuinfo The RISC-V specs are permissive in what they allow as the ISA string, but how we output this to userspace in /proc/cpuinfo is quasi uABI. Formalise this as part of the uABI, by documenting the list of rules we use at this point in time. Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20221205144525.2148448-4-conor.dooley@microchip.com Signed-off-by: Palmer Dabbelt --- Documentation/riscv/uabi.rst | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Documentation/riscv/uabi.rst b/Documentation/riscv/uabi.rst index 21a82cfb6c4d..2ebec4c52230 100644 --- a/Documentation/riscv/uabi.rst +++ b/Documentation/riscv/uabi.rst @@ -3,4 +3,46 @@ RISC-V Linux User ABI ===================== +ISA string ordering in /proc/cpuinfo +------------------------------------ + +The canonical order of ISA extension names in the ISA string is defined in +chapter 27 of the unprivileged specification. +The specification uses vague wording, such as should, when it comes to ordering, +so for our purposes the following rules apply: + +#. Single-letter extensions come first, in canonical order. + The canonical order is "IMAFDQLCBKJTPVH". + +#. All multi-letter extensions will be separated from other extensions by an + underscore. + +#. Additional standard extensions (starting with 'Z') will be sorted after + single-letter extensions and before any higher-privileged extensions. + +#. For additional standard extensions, the first letter following the 'Z' + conventionally indicates the most closely related alphabetical + extension category. If multiple 'Z' extensions are named, they will be ordered + first by category, in canonical order, as listed above, then alphabetically + within a category. + +#. Standard supervisor-level extensions (starting with 'S') will be listed + after standard unprivileged extensions. If multiple supervisor-level + extensions are listed, they will be ordered alphabetically. + +#. Standard machine-level extensions (starting with 'Zxm') will be listed + after any lower-privileged, standard extensions. If multiple machine-level + extensions are listed, they will be ordered alphabetically. + +#. Non-standard extensions (starting with 'X') will be listed after all standard + extensions. If multiple non-standard extensions are listed, they will be + ordered alphabetically. + +An example string following the order is:: + + rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux + +Misaligned accesses +------------------- + Misaligned accesses are supported in userspace, but they may perform poorly. From 6be1ff430dab9fc047762b10b2c9669399ea1f37 Mon Sep 17 00:00:00 2001 From: Guo Ren Date: Tue, 10 Jan 2023 03:04:19 -0500 Subject: [PATCH 17/67] riscv: pgtable: Fixup comment for KERN_VIRT_SIZE KERN_VIRT_SIZE is 1/4 of the entries of the page global directory, not half. Fixes: f7ae02333d13 ("riscv: Move KASAN mapping next to the kernel mapping") Signed-off-by: Guo Ren Signed-off-by: Guo Ren Cc: Alexandre Ghiti Reviewed-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20230110080419.931185-1-guoren@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 4eba9a98d0e3..2a88362dffa5 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -31,7 +31,7 @@ #define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) /* - * Half of the kernel address space (half of the entries of the page global + * Half of the kernel address space (1/4 of the entries of the page global * directory) is for the direct mapping. */ #define KERN_VIRT_SIZE ((PTRS_PER_PGD / 2 * PGDIR_SIZE) / 2) From 451fb217cdca0ead3f396f00f8da331401344950 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 11 Jan 2023 09:15:33 +0100 Subject: [PATCH 18/67] RISC-V: Kconfig: Remove trailing whitespace Remove trailing whitespace that hurts my eyes. Signed-off-by: Geert Uytterhoeven Acked-by: Conor Dooley Link: https://lore.kernel.org/r/080aa959266ad842a8e7efca7111f1350c6a065a.1673424858.git.geert+renesas@glider.be Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig.socs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index d36a5f39f13a..659140309157 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -34,7 +34,7 @@ config SOC_STARFIVE config ARCH_VIRT def_bool SOC_VIRT - + config SOC_VIRT bool "QEMU Virt Machine" select CLINT_TIMER if RISCV_M_MODE From 9d5567ccf96fb2f1eb83d531eee23ead4aa8f2a3 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Fri, 13 Jan 2023 22:22:05 +0100 Subject: [PATCH 19/67] riscv: fix jal offsets in patched alternatives Alternatives live in a different section, so offsets used by jal instruction will point to wrong locations after the patch got applied. Similar to arm64, adjust the location to consider that offset. Co-developed-by: Heiko Stuebner Signed-off-by: Jisheng Zhang Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20230113212205.3534622-1-heiko@sntech.de Fixes: 27c653c06505 ("RISC-V: fix auipc-jalr addresses in patched alternatives") Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/insn.h | 27 +++++++++++++++++++++++++++ arch/riscv/kernel/alternative.c | 27 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 98453535324a..25ef9c0b19e7 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -291,6 +291,33 @@ static __always_inline bool riscv_insn_is_branch(u32 code) (RVC_X(x_, RVC_B_IMM_7_6_OPOFF, RVC_B_IMM_7_6_MASK) << RVC_B_IMM_7_6_OFF) | \ (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) +/* + * Get the immediate from a J-type instruction. + * + * @insn: instruction to process + * Return: immediate + */ +static inline s32 riscv_insn_extract_jtype_imm(u32 insn) +{ + return RV_EXTRACT_JTYPE_IMM(insn); +} + +/* + * Update a J-type instruction with an immediate value. + * + * @insn: pointer to the jtype instruction + * @imm: the immediate to insert into the instruction + */ +static inline void riscv_insn_insert_jtype_imm(u32 *insn, s32 imm) +{ + /* drop the old IMMs, all jal IMM bits sit at 31:12 */ + *insn &= ~GENMASK(31, 12); + *insn |= (RV_X(imm, RV_J_IMM_10_1_OFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OPOFF) | + (RV_X(imm, RV_J_IMM_11_OFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OPOFF) | + (RV_X(imm, RV_J_IMM_19_12_OFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OPOFF) | + (RV_X(imm, RV_J_IMM_SIGN_OFF, 1) << RV_J_IMM_SIGN_OPOFF); +} + /* * Put together one immediate from a U-type and I-type instruction pair. * diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 6212ea0eed72..3d4f1f32c7f6 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -79,6 +79,21 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, patch_text_nosync(ptr, call, sizeof(u32) * 2); } +static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset) +{ + s32 imm; + + /* get and adjust new target address */ + imm = riscv_insn_extract_jtype_imm(jal_insn); + imm -= patch_offset; + + /* update instruction */ + riscv_insn_insert_jtype_imm(&jal_insn, imm); + + /* patch the call place again */ + patch_text_nosync(ptr, &jal_insn, sizeof(u32)); +} + void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, int patch_offset) { @@ -106,6 +121,18 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), insn, insn2, patch_offset); } + + if (riscv_insn_is_jal(insn)) { + s32 imm = riscv_insn_extract_jtype_imm(insn); + + /* Don't modify jumps inside the alternative block */ + if ((alt_ptr + i * sizeof(u32) + imm) >= alt_ptr && + (alt_ptr + i * sizeof(u32) + imm) < (alt_ptr + len)) + continue; + + riscv_alternative_fix_jal(alt_ptr + i * sizeof(u32), + insn, patch_offset); + } } } From 285b6a18daf1358e70a4c842884d9ff2d2fe53e2 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Wed, 18 Jan 2023 15:22:52 +0100 Subject: [PATCH 20/67] RISC-V: Fix do_notify_resume / do_work_pending prototype Commit b0f4c74eadbf ("RISC-V: Fix unannoted hardirqs-on in return to userspace slow-path") renamed the do_notify_resume function to do_work_pending but did not change the prototype in signal.h Do that now, as the original function does not exist anymore. Fixes: b0f4c74eadbf ("RISC-V: Fix unannoted hardirqs-on in return to userspace slow-path") Signed-off-by: Heiko Stuebner Reviewed-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230118142252.337103-1-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/signal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/signal.h b/arch/riscv/include/asm/signal.h index 532c29ef0376..956ae0a01bad 100644 --- a/arch/riscv/include/asm/signal.h +++ b/arch/riscv/include/asm/signal.h @@ -7,6 +7,6 @@ #include asmlinkage __visible -void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags); +void do_work_pending(struct pt_regs *regs, unsigned long thread_info_flags); #endif From 56e0790c7f9e59ba6a0f4b59981d1d6fbf43efb0 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 13 Jan 2023 22:23:00 +0100 Subject: [PATCH 21/67] RISC-V: add infrastructure to allow different str* implementations Depending on supported extensions on specific RISC-V cores, optimized str* functions might make sense. This adds basic infrastructure to allow patching the function calls via alternatives later on. The Linux kernel provides standard implementations for string functions but when architectures want to extend them, they need to provide their own. The added generic string functions are done in assembler (taken from disassembling the main-kernel functions for now) to allow us to control the used registers and extend them with optimized variants. This doesn't override the compiler's use of builtin replacements. So still first of all the compiler will select if a builtin will be better suitable i.e. for known strings. For all regular cases we will want to later select possible optimized variants and in the worst case fall back to the generic implemention added with this change. Reviewed-by: Andrew Jones Signed-off-by: Heiko Stuebner Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230113212301.3534711-2-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/string.h | 10 ++++++++ arch/riscv/kernel/riscv_ksyms.c | 3 +++ arch/riscv/lib/Makefile | 3 +++ arch/riscv/lib/strcmp.S | 36 +++++++++++++++++++++++++++++ arch/riscv/lib/strlen.S | 28 ++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 41 +++++++++++++++++++++++++++++++++ arch/riscv/purgatory/Makefile | 13 +++++++++++ 7 files changed, 134 insertions(+) create mode 100644 arch/riscv/lib/strcmp.S create mode 100644 arch/riscv/lib/strlen.S create mode 100644 arch/riscv/lib/strncmp.S diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 909049366555..a96b1fea24fe 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -18,6 +18,16 @@ extern asmlinkage void *__memcpy(void *, const void *, size_t); #define __HAVE_ARCH_MEMMOVE extern asmlinkage void *memmove(void *, const void *, size_t); extern asmlinkage void *__memmove(void *, const void *, size_t); + +#define __HAVE_ARCH_STRCMP +extern asmlinkage int strcmp(const char *cs, const char *ct); + +#define __HAVE_ARCH_STRLEN +extern asmlinkage __kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); + /* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) #define memcpy(dst, src, len) __memcpy(dst, src, len) diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c index 5ab1c7e1a6ed..a72879b4249a 100644 --- a/arch/riscv/kernel/riscv_ksyms.c +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -12,6 +12,9 @@ EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); EXPORT_SYMBOL(memmove); +EXPORT_SYMBOL(strcmp); +EXPORT_SYMBOL(strlen); +EXPORT_SYMBOL(strncmp); EXPORT_SYMBOL(__memset); EXPORT_SYMBOL(__memcpy); EXPORT_SYMBOL(__memmove); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 25d5c9664e57..6c74b0bedd60 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -3,6 +3,9 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o lib-y += memmove.o +lib-y += strcmp.o +lib-y += strlen.o +lib-y += strncmp.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o diff --git a/arch/riscv/lib/strcmp.S b/arch/riscv/lib/strcmp.S new file mode 100644 index 000000000000..8babd712b958 --- /dev/null +++ b/arch/riscv/lib/strcmp.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include + +/* int strcmp(const char *cs, const char *ct) */ +SYM_FUNC_START(strcmp) + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1 + */ +1: + lbu t0, 0(a0) + lbu t1, 0(a1) + addi a0, a0, 1 + addi a1, a1, 1 + bne t0, t1, 2f + bnez t0, 1b + li a0, 0 + ret +2: + /* + * strcmp only needs to return (< 0, 0, > 0) values + * not necessarily -1, 0, +1 + */ + sub a0, t0, t1 + ret +SYM_FUNC_END(strcmp) diff --git a/arch/riscv/lib/strlen.S b/arch/riscv/lib/strlen.S new file mode 100644 index 000000000000..0a3b11853efd --- /dev/null +++ b/arch/riscv/lib/strlen.S @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include + +/* int strlen(const char *s) */ +SYM_FUNC_START(strlen) + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers: + * t0, t1 + */ + mv t1, a0 +1: + lbu t0, 0(t1) + beqz t0, 2f + addi t1, t1, 1 + j 1b +2: + sub a0, t1, a0 + ret +SYM_FUNC_END(strlen) diff --git a/arch/riscv/lib/strncmp.S b/arch/riscv/lib/strncmp.S new file mode 100644 index 000000000000..1f644d0a93f6 --- /dev/null +++ b/arch/riscv/lib/strncmp.S @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include + +/* int strncmp(const char *cs, const char *ct, size_t count) */ +SYM_FUNC_START(strncmp) + /* + * Returns + * a0 - comparison result, value like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2 + */ + li t2, 0 +1: + beq a2, t2, 2f + lbu t0, 0(a0) + lbu t1, 0(a1) + addi a0, a0, 1 + addi a1, a1, 1 + bne t0, t1, 3f + addi t2, t2, 1 + bnez t0, 1b +2: + li a0, 0 + ret +3: + /* + * strncmp only needs to return (< 0, 0, > 0) values + * not necessarily -1, 0, +1 + */ + sub a0, t0, t1 + ret +SYM_FUNC_END(strncmp) diff --git a/arch/riscv/purgatory/Makefile b/arch/riscv/purgatory/Makefile index dd58e1d99397..d16bf715a586 100644 --- a/arch/riscv/purgatory/Makefile +++ b/arch/riscv/purgatory/Makefile @@ -2,6 +2,7 @@ OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o +purgatory-y += strcmp.o strlen.o strncmp.o targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) @@ -18,6 +19,15 @@ $(obj)/memcpy.o: $(srctree)/arch/riscv/lib/memcpy.S FORCE $(obj)/memset.o: $(srctree)/arch/riscv/lib/memset.S FORCE $(call if_changed_rule,as_o_S) +$(obj)/strcmp.o: $(srctree)/arch/riscv/lib/strcmp.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strlen.o: $(srctree)/arch/riscv/lib/strlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE + $(call if_changed_rule,as_o_S) + $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c) @@ -77,6 +87,9 @@ CFLAGS_ctype.o += $(PURGATORY_CFLAGS) AFLAGS_REMOVE_entry.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memcpy.o += -Wa,-gdwarf-2 AFLAGS_REMOVE_memset.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strcmp.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strlen.o += -Wa,-gdwarf-2 +AFLAGS_REMOVE_strncmp.o += -Wa,-gdwarf-2 $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld) From b6fcdb191e36f82336f9b5e126d51c02e7323480 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 13 Jan 2023 22:23:01 +0100 Subject: [PATCH 22/67] RISC-V: add zbb support to string functions Add handling for ZBB extension and add support for using it as a variant for optimized string functions. Support for the Zbb-str-variants is limited to the GNU-assembler for now, as LLVM has not yet acquired the functionality to selectively change the arch option in assembler code. This is still under review at https://reviews.llvm.org/D123515 Co-developed-by: Christoph Muellner Signed-off-by: Christoph Muellner Signed-off-by: Heiko Stuebner Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230113212301.3534711-3-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 24 ++++++ arch/riscv/include/asm/errata_list.h | 3 +- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/kernel/cpu.c | 1 + arch/riscv/kernel/cpufeature.c | 18 +++++ arch/riscv/lib/strcmp.S | 85 ++++++++++++++++++++++ arch/riscv/lib/strlen.S | 105 +++++++++++++++++++++++++++ arch/riscv/lib/strncmp.S | 98 +++++++++++++++++++++++++ 8 files changed, 334 insertions(+), 1 deletion(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index e2b656043abf..7c814fbf9527 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -416,6 +416,30 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_IS_GNU + +config RISCV_ISA_ZBB + bool "Zbb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBB + depends on !XIP_KERNEL && MMU + select RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBB + extension (basic bit manipulation) and enable its usage. + + The Zbb extension provides instructions to accelerate a number + of bit-specific operations (count bit population, sign extending, + bitrotation, etc). + + If you don't know what to do here, say Y. + config TOOLCHAIN_HAS_ZICBOM bool default y diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index 4180312d2a70..95e626b7281e 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -24,7 +24,8 @@ #define CPUFEATURE_SVPBMT 0 #define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 +#define CPUFEATURE_ZBB 2 +#define CPUFEATURE_NUMBER 3 #ifdef __ASSEMBLY__ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 57439da71c77..462d6cde9bac 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -58,6 +58,7 @@ enum riscv_isa_ext_id { RISCV_ISA_EXT_SSTC, RISCV_ISA_EXT_SVINVAL, RISCV_ISA_EXT_SVPBMT, + RISCV_ISA_EXT_ZBB, RISCV_ISA_EXT_ZICBOM, RISCV_ISA_EXT_ZIHINTPAUSE, RISCV_ISA_EXT_ID_MAX diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 0bf1c7f663fc..420228e219f7 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -185,6 +185,7 @@ arch_initcall(riscv_cpuinfo_init); * New entries to this struct should follow the ordering rules described above. */ static struct riscv_isa_ext_data isa_ext_arr[] = { + __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index dde0e91d7668..9899806cef29 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -227,6 +227,7 @@ void __init riscv_fill_hwcap(void) SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); + SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB); SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); } @@ -302,6 +303,20 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) return true; } +static bool __init_or_module cpufeature_probe_zbb(unsigned int stage) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_ZBB)) + return false; + + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + return false; + + if (!riscv_isa_extension_available(NULL, ZBB)) + return false; + + return true; +} + /* * Probe presence of individual extensions. * @@ -320,6 +335,9 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage) if (cpufeature_probe_zicbom(stage)) cpu_req_feature |= BIT(CPUFEATURE_ZICBOM); + if (cpufeature_probe_zbb(stage)) + cpu_req_feature |= BIT(CPUFEATURE_ZBB); + return cpu_req_feature; } diff --git a/arch/riscv/lib/strcmp.S b/arch/riscv/lib/strcmp.S index 8babd712b958..8148b6418f61 100644 --- a/arch/riscv/lib/strcmp.S +++ b/arch/riscv/lib/strcmp.S @@ -3,9 +3,14 @@ #include #include #include +#include +#include /* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) + + ALTERNATIVE("nop", "j strcmp_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - comparison result, value like strcmp @@ -33,4 +38,84 @@ SYM_FUNC_START(strcmp) */ sub a0, t0, t1 ret + +/* + * Variant of strcmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +strcmp_zbb: + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, value like strcmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * + * Clobbers + * t0, t1, t2, t3, t4, t5 + */ + + or t2, a0, a1 + li t4, -1 + and t2, t2, SZREG-1 + bnez t2, 3f + + /* Main loop for aligned string. */ + .p2align 3 +1: + REG_L t0, 0(a0) + REG_L t1, 0(a1) + orc.b t3, t0 + bne t3, t4, 2f + addi a0, a0, SZREG + addi a1, a1, SZREG + beq t0, t1, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 t0, t0 + rev8 t1, t1 +#endif + + /* Synthesize (t0 >= t1) ? 1 : -1 in a branchless sequence. */ + sltu a0, t0, t1 + neg a0, a0 + ori a0, a0, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne t0, t1, 3f + + /* Otherwise, strings are equal. */ + li a0, 0 + ret + + /* Simple loop for misaligned strings. */ + .p2align 3 +3: + lbu t0, 0(a0) + lbu t1, 0(a1) + addi a0, a0, 1 + addi a1, a1, 1 + bne t0, t1, 4f + bnez t0, 3b + +4: + sub a0, t0, t1 + ret + +.option pop +#endif SYM_FUNC_END(strcmp) diff --git a/arch/riscv/lib/strlen.S b/arch/riscv/lib/strlen.S index 0a3b11853efd..0f9dbf93301a 100644 --- a/arch/riscv/lib/strlen.S +++ b/arch/riscv/lib/strlen.S @@ -3,9 +3,14 @@ #include #include #include +#include +#include /* int strlen(const char *s) */ SYM_FUNC_START(strlen) + + ALTERNATIVE("nop", "j strlen_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - string length @@ -25,4 +30,104 @@ SYM_FUNC_START(strlen) 2: sub a0, t1, a0 ret + +/* + * Variant of strlen using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +strlen_zbb: + +#ifdef CONFIG_CPU_BIG_ENDIAN +# define CZ clz +# define SHIFT sll +#else +# define CZ ctz +# define SHIFT srl +#endif + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - string length + * + * Parameters + * a0 - String to measure + * + * Clobbers + * t0, t1, t2, t3 + */ + + /* Number of irrelevant bytes in the first word. */ + andi t2, a0, SZREG-1 + + /* Align pointer. */ + andi t0, a0, -SZREG + + li t3, SZREG + sub t3, t3, t2 + slli t2, t2, 3 + + /* Get the first word. */ + REG_L t1, 0(t0) + + /* + * Shift away the partial data we loaded to remove the irrelevant bytes + * preceding the string with the effect of adding NUL bytes at the + * end of the string's first word. + */ + SHIFT t1, t1, t2 + + /* Convert non-NUL into 0xff and NUL into 0x00. */ + orc.b t1, t1 + + /* Convert non-NUL into 0x00 and NUL into 0xff. */ + not t1, t1 + + /* + * Search for the first set bit (corresponding to a NUL byte in the + * original chunk). + */ + CZ t1, t1 + + /* + * The first chunk is special: compare against the number + * of valid bytes in this chunk. + */ + srli a0, t1, 3 + bgtu t3, a0, 3f + + /* Prepare for the word comparison loop. */ + addi t2, t0, SZREG + li t3, -1 + + /* + * Our critical loop is 4 instructions and processes data in + * 4 byte or 8 byte chunks. + */ + .p2align 3 +1: + REG_L t1, SZREG(t0) + addi t0, t0, SZREG + orc.b t1, t1 + beq t1, t3, 1b +2: + not t1, t1 + CZ t1, t1 + + /* Get number of processed words. */ + sub t2, t0, t2 + + /* Add number of characters in the first word. */ + add a0, a0, t2 + srli t1, t1, 3 + + /* Add number of characters in the last word. */ + add a0, a0, t1 +3: + ret + +.option pop +#endif SYM_FUNC_END(strlen) diff --git a/arch/riscv/lib/strncmp.S b/arch/riscv/lib/strncmp.S index 1f644d0a93f6..7940ddab2d48 100644 --- a/arch/riscv/lib/strncmp.S +++ b/arch/riscv/lib/strncmp.S @@ -3,9 +3,14 @@ #include #include #include +#include +#include /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) + + ALTERNATIVE("nop", "j strncmp_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + /* * Returns * a0 - comparison result, value like strncmp @@ -38,4 +43,97 @@ SYM_FUNC_START(strncmp) */ sub a0, t0, t1 ret + +/* + * Variant of strncmp using the ZBB extension if available + */ +#ifdef CONFIG_RISCV_ISA_ZBB +strncmp_zbb: + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - comparison result, like strncmp + * + * Parameters + * a0 - string1 + * a1 - string2 + * a2 - number of characters to compare + * + * Clobbers + * t0, t1, t2, t3, t4, t5, t6 + */ + + or t2, a0, a1 + li t5, -1 + and t2, t2, SZREG-1 + add t4, a0, a2 + bnez t2, 4f + + /* Adjust limit for fast-path. */ + andi t6, t4, -SZREG + + /* Main loop for aligned string. */ + .p2align 3 +1: + bgt a0, t6, 3f + REG_L t0, 0(a0) + REG_L t1, 0(a1) + orc.b t3, t0 + bne t3, t5, 2f + addi a0, a0, SZREG + addi a1, a1, SZREG + beq t0, t1, 1b + + /* + * Words don't match, and no null byte in the first + * word. Get bytes in big-endian order and compare. + */ +#ifndef CONFIG_CPU_BIG_ENDIAN + rev8 t0, t0 + rev8 t1, t1 +#endif + + /* Synthesize (t0 >= t1) ? 1 : -1 in a branchless sequence. */ + sltu a0, t0, t1 + neg a0, a0 + ori a0, a0, 1 + ret + +2: + /* + * Found a null byte. + * If words don't match, fall back to simple loop. + */ + bne t0, t1, 3f + + /* Otherwise, strings are equal. */ + li a0, 0 + ret + + /* Simple loop for misaligned strings. */ +3: + /* Restore limit for slow-path. */ + .p2align 3 +4: + bge a0, t4, 6f + lbu t0, 0(a0) + lbu t1, 0(a1) + addi a0, a0, 1 + addi a1, a1, 1 + bne t0, t1, 5f + bnez t0, 4b + +5: + sub a0, t0, t1 + ret + +6: + li a0, 0 + ret + +.option pop +#endif SYM_FUNC_END(strncmp) From abcc445acdbeb561e881fcbd8e1dc4684628cc9b Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:44 +0800 Subject: [PATCH 23/67] riscv: move riscv_noncoherent_supported() out of ZICBOM probe It's a bit weird to call riscv_noncoherent_supported() each time when insmoding a module. Move the calling out of feature patch func. Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-2-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpufeature.c | 1 - arch/riscv/kernel/setup.c | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index dde0e91d7668..62443fd32fa7 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -298,7 +298,6 @@ static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) if (!riscv_isa_extension_available(NULL, ZICBOM)) return false; - riscv_noncoherent_supported(); return true; } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 86acd690d529..376d2827e736 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -300,6 +300,9 @@ void __init setup_arch(char **cmdline_p) riscv_init_cbom_blocksize(); riscv_fill_hwcap(); apply_boot_alternatives(); + if (IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM) && + riscv_isa_extension_available(NULL, ZICBOM)) + riscv_noncoherent_supported(); } static int __init topology_init(void) From 191b27c7c0e8c1a587832f8680132564743af76c Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:45 +0800 Subject: [PATCH 24/67] riscv: cpufeature: detect RISCV_ALTERNATIVES_EARLY_BOOT earlier Currently riscv_cpufeature_patch_func() does nothing at the RISCV_ALTERNATIVES_EARLY_BOOT stage. Add a check to detect whether we are in this stage and exit early. This will allow us to use riscv_cpufeature_patch_func() for scanning of all ISA extensions. Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Reviewed-by: Heiko Stuebner Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-3-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpufeature.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 62443fd32fa7..59e20cad1b3d 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -330,6 +330,9 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *alt; u32 tmp; + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + return; + for (alt = begin; alt < end; alt++) { if (alt->vendor_id != 0) continue; From d8a3d8a75206d3dfba64b6d4468b7cf44d4eb4d7 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:46 +0800 Subject: [PATCH 25/67] riscv: hwcap: make ISA extension ids can be used in asm So that ISA extensions can be used in assembly files, convert the multi-letter RISC-V ISA extension IDs enums to macros. In order to make them visible, move the #ifndef __ASSEMBLY__ guard to a later point in the header Signed-off-by: Jisheng Zhang Reviewed-by: Heiko Stuebner Reviewed-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-4-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 45 ++++++++++++++++------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 57439da71c77..8e0ee841fa77 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -12,20 +12,6 @@ #include #include -#ifndef __ASSEMBLY__ -#include -/* - * This yields a mask that user programs can use to figure out what - * instruction set this cpu supports. - */ -#define ELF_HWCAP (elf_hwcap) - -enum { - CAP_HWCAP = 1, -}; - -extern unsigned long elf_hwcap; - #define RISCV_ISA_EXT_a ('a' - 'a') #define RISCV_ISA_EXT_c ('c' - 'a') #define RISCV_ISA_EXT_d ('d' - 'a') @@ -46,23 +32,34 @@ extern unsigned long elf_hwcap; #define RISCV_ISA_EXT_BASE 26 /* - * This enum represent the logical ID for each multi-letter RISC-V ISA extension. + * These macros represent the logical ID for each multi-letter RISC-V ISA extension. * The logical ID should start from RISCV_ISA_EXT_BASE and must not exceed * RISCV_ISA_EXT_MAX. 0-25 range is reserved for single letter * extensions while all the multi-letter extensions should define the next * available logical extension id. * Entries are sorted alphabetically. */ -enum riscv_isa_ext_id { - RISCV_ISA_EXT_SSCOFPMF = RISCV_ISA_EXT_BASE, - RISCV_ISA_EXT_SSTC, - RISCV_ISA_EXT_SVINVAL, - RISCV_ISA_EXT_SVPBMT, - RISCV_ISA_EXT_ZICBOM, - RISCV_ISA_EXT_ZIHINTPAUSE, - RISCV_ISA_EXT_ID_MAX +#define RISCV_ISA_EXT_SSCOFPMF 26 +#define RISCV_ISA_EXT_SSTC 27 +#define RISCV_ISA_EXT_SVINVAL 28 +#define RISCV_ISA_EXT_SVPBMT 29 +#define RISCV_ISA_EXT_ZICBOM 30 +#define RISCV_ISA_EXT_ZIHINTPAUSE 31 + +#ifndef __ASSEMBLY__ +#include +/* + * This yields a mask that user programs can use to figure out what + * instruction set this cpu supports. + */ +#define ELF_HWCAP (elf_hwcap) + +enum { + CAP_HWCAP = 1, }; -static_assert(RISCV_ISA_EXT_ID_MAX <= RISCV_ISA_EXT_MAX); + +extern unsigned long elf_hwcap; + /* * This enum represents the logical ID for each RISC-V ISA extension static From 4bf8860760d9930019e4839fb4e8adc1e0bf1f91 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:47 +0800 Subject: [PATCH 26/67] riscv: cpufeature: extend riscv_cpufeature_patch_func to all ISA extensions riscv_cpufeature_patch_func() currently only scans a limited set of cpufeatures, explicitly defined with macros. Extend it to probe for all ISA extensions. Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-5-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/errata_list.h | 9 ++-- arch/riscv/kernel/cpufeature.c | 64 ++++------------------------ 2 files changed, 11 insertions(+), 62 deletions(-) diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index 4180312d2a70..274c6f889602 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -7,6 +7,7 @@ #include #include +#include #include #ifdef CONFIG_ERRATA_SIFIVE @@ -22,10 +23,6 @@ #define ERRATA_THEAD_NUMBER 3 #endif -#define CPUFEATURE_SVPBMT 0 -#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_NUMBER 2 - #ifdef __ASSEMBLY__ #define ALT_INSN_FAULT(x) \ @@ -55,7 +52,7 @@ asm(ALTERNATIVE("sfence.vma %0", "sfence.vma", SIFIVE_VENDOR_ID, \ #define ALT_SVPBMT(_val, prot) \ asm(ALTERNATIVE_2("li %0, 0\t\nnop", \ "li %0, %1\t\nslli %0,%0,%3", 0, \ - CPUFEATURE_SVPBMT, CONFIG_RISCV_ISA_SVPBMT, \ + RISCV_ISA_EXT_SVPBMT, CONFIG_RISCV_ISA_SVPBMT, \ "li %0, %2\t\nslli %0,%0,%4", THEAD_VENDOR_ID, \ ERRATA_THEAD_PBMT, CONFIG_ERRATA_THEAD_PBMT) \ : "=r"(_val) \ @@ -129,7 +126,7 @@ asm volatile(ALTERNATIVE_2( \ "add a0, a0, %0\n\t" \ "2:\n\t" \ "bltu a0, %2, 3b\n\t" \ - "nop", 0, CPUFEATURE_ZICBOM, CONFIG_RISCV_ISA_ZICBOM, \ + "nop", 0, RISCV_ISA_EXT_ZICBOM, CONFIG_RISCV_ISA_ZICBOM, \ "mv a0, %1\n\t" \ "j 2f\n\t" \ "3:\n\t" \ diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 59e20cad1b3d..6193f401f0c5 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -276,59 +276,11 @@ void __init riscv_fill_hwcap(void) } #ifdef CONFIG_RISCV_ALTERNATIVE -static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage) -{ - if (!IS_ENABLED(CONFIG_RISCV_ISA_SVPBMT)) - return false; - - if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) - return false; - - return riscv_isa_extension_available(NULL, SVPBMT); -} - -static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) -{ - if (!IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM)) - return false; - - if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) - return false; - - if (!riscv_isa_extension_available(NULL, ZICBOM)) - return false; - - return true; -} - -/* - * Probe presence of individual extensions. - * - * This code may also be executed before kernel relocation, so we cannot use - * addresses generated by the address-of operator as they won't be valid in - * this context. - * Tests, unless otherwise required, are to be added in alphabetical order. - */ -static u32 __init_or_module cpufeature_probe(unsigned int stage) -{ - u32 cpu_req_feature = 0; - - if (cpufeature_probe_svpbmt(stage)) - cpu_req_feature |= BIT(CPUFEATURE_SVPBMT); - - if (cpufeature_probe_zicbom(stage)) - cpu_req_feature |= BIT(CPUFEATURE_ZICBOM); - - return cpu_req_feature; -} - void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned int stage) { - u32 cpu_req_feature = cpufeature_probe(stage); struct alt_entry *alt; - u32 tmp; if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) return; @@ -336,18 +288,18 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, for (alt = begin; alt < end; alt++) { if (alt->vendor_id != 0) continue; - if (alt->errata_id >= CPUFEATURE_NUMBER) { - WARN(1, "This feature id:%d is not in kernel cpufeature list", + if (alt->errata_id >= RISCV_ISA_EXT_MAX) { + WARN(1, "This extension id:%d is not in ISA extension list", alt->errata_id); continue; } - tmp = (1U << alt->errata_id); - if (cpu_req_feature & tmp) { - patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); - riscv_alternative_fix_offsets(alt->old_ptr, alt->alt_len, - alt->old_ptr - alt->alt_ptr); - } + if (!__riscv_isa_extension_available(NULL, alt->errata_id)) + continue; + + patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); + riscv_alternative_fix_offsets(alt->old_ptr, alt->alt_len, + alt->old_ptr - alt->alt_ptr); } } #endif From bdda5d554e43f21d06e0b7485e6241ca46c26b99 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:48 +0800 Subject: [PATCH 27/67] riscv: introduce riscv_has_extension_[un]likely() Generally, riscv ISA extensions are fixed for any specific hardware platform, so a hart's features won't change after booting. This chacteristic makes it straightforward to use a static branch to check if a specific ISA extension is supported or not to optimize performance. However, some ISA extensions such as SVPBMT and ZICBOM are handled via. the alternative sequences. Basically, for ease of maintenance, we prefer to use static branches in C code, but recently, Samuel found that the static branch usage in cpu_relax() breaks building with CONFIG_CC_OPTIMIZE_FOR_SIZE[1]. As Samuel pointed out, "Having a static branch in cpu_relax() is problematic because that function is widely inlined, including in some quite complex functions like in the VDSO. A quick measurement shows this static branch is responsible by itself for around 40% of the jump table." Samuel's findings pointed out one of a few downsides of static branches usage in C code to handle ISA extensions detected at boot time: static branch's metadata in the __jump_table section, which is not discarded after ISA extensions are finalized, wastes some space. I want to try to solve the issue for all possible dynamic handling of ISA extensions at boot time. Inspired by Mark[2], this patch introduces riscv_has_extension_*() helpers, which work like static branches but are patched using alternatives, thus the metadata can be freed after patching. Link: https://lore.kernel.org/linux-riscv/20220922060958.44203-1-samuel@sholland.org/ [1] Link: https://lore.kernel.org/linux-arm-kernel/20220912162210.3626215-8-mark.rutland@arm.com/ [2] Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-6-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 8e0ee841fa77..411ef0fb5c4b 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -8,6 +8,7 @@ #ifndef _ASM_RISCV_HWCAP_H #define _ASM_RISCV_HWCAP_H +#include #include #include #include @@ -99,6 +100,42 @@ static __always_inline int riscv_isa_ext2key(int num) } } +static __always_inline bool +riscv_has_extension_likely(const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_EXT_MAX, + "ext must be < RISCV_ISA_EXT_MAX"); + + asm_volatile_goto( + ALTERNATIVE("j %l[l_no]", "nop", 0, %[ext], 1) + : + : [ext] "i" (ext) + : + : l_no); + + return true; +l_no: + return false; +} + +static __always_inline bool +riscv_has_extension_unlikely(const unsigned long ext) +{ + compiletime_assert(ext < RISCV_ISA_EXT_MAX, + "ext must be < RISCV_ISA_EXT_MAX"); + + asm_volatile_goto( + ALTERNATIVE("nop", "j %l[l_yes]", 0, %[ext], 1) + : + : [ext] "i" (ext) + : + : l_yes); + + return false; +l_yes: + return true; +} + unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap); #define riscv_isa_extension_mask(ext) BIT_MASK(RISCV_ISA_EXT_##ext) From 702e64550b12cf36089672af8f110660da7f847e Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:49 +0800 Subject: [PATCH 28/67] riscv: fpu: switch has_fpu() to riscv_has_extension_likely() Switch has_fpu() from static branch to the new helper riscv_has_extension_likely(). Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Reviewed-by: Heiko Stuebner Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-7-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/switch_to.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 11463489fec6..60f8ca01d36e 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -59,7 +59,8 @@ static inline void __switch_to_aux(struct task_struct *prev, static __always_inline bool has_fpu(void) { - return static_branch_likely(&riscv_isa_ext_keys[RISCV_ISA_EXT_KEY_FPU]); + return riscv_has_extension_likely(RISCV_ISA_EXT_f) || + riscv_has_extension_likely(RISCV_ISA_EXT_d); } #else static __always_inline bool has_fpu(void) { return false; } From e0c267e03b0c77c9ac79ac08eada41ba8eb1b95f Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:50 +0800 Subject: [PATCH 29/67] riscv: module: move find_section to module.h Move find_section() to module.h so that the implementation can be shared by the alternatives code. This will allow us to use alternatives in the vdso. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Signed-off-by: Jisheng Zhang Link: https://lore.kernel.org/r/20230128172856.3814-8-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/module.h | 16 ++++++++++++++++ arch/riscv/kernel/module.c | 15 --------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/riscv/include/asm/module.h b/arch/riscv/include/asm/module.h index 76aa96a9fc08..0f3baaa6a9a8 100644 --- a/arch/riscv/include/asm/module.h +++ b/arch/riscv/include/asm/module.h @@ -5,6 +5,7 @@ #define _ASM_RISCV_MODULE_H #include +#include struct module; unsigned long module_emit_got_entry(struct module *mod, unsigned long val); @@ -111,4 +112,19 @@ static inline struct plt_entry *get_plt_entry(unsigned long val, #endif /* CONFIG_MODULE_SECTIONS */ +static inline const Elf_Shdr *find_section(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, + const char *name) +{ + const Elf_Shdr *s, *se; + const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + + for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { + if (strcmp(name, secstrs + s->sh_name) == 0) + return s; + } + + return NULL; +} + #endif /* _ASM_RISCV_MODULE_H */ diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 91fe16bfaa07..76f4b9c2ec5b 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -429,21 +429,6 @@ void *module_alloc(unsigned long size) } #endif -static const Elf_Shdr *find_section(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - const char *name) -{ - const Elf_Shdr *s, *se; - const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; - - for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { - if (strcmp(name, secstrs + s->sh_name) == 0) - return s; - } - - return NULL; -} - int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) From 1bc400ffb52b460eedc51ebd9b7753b7f2314878 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Sun, 29 Jan 2023 01:28:51 +0800 Subject: [PATCH 30/67] riscv: module: Add ADD16 and SUB16 rela types To prepare for 16-bit relocation types to be emitted in alternatives add support for ADD16 and SUB16. Signed-off-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-9-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/module.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 76f4b9c2ec5b..7c651d55fcbd 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -268,6 +268,13 @@ static int apply_r_riscv_align_rela(struct module *me, u32 *location, return -EINVAL; } +static int apply_r_riscv_add16_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u16 *)location += (u16)v; + return 0; +} + static int apply_r_riscv_add32_rela(struct module *me, u32 *location, Elf_Addr v) { @@ -282,6 +289,13 @@ static int apply_r_riscv_add64_rela(struct module *me, u32 *location, return 0; } +static int apply_r_riscv_sub16_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *(u16 *)location -= (u16)v; + return 0; +} + static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, Elf_Addr v) { @@ -315,8 +329,10 @@ static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, [R_RISCV_CALL] = apply_r_riscv_call_rela, [R_RISCV_RELAX] = apply_r_riscv_relax_rela, [R_RISCV_ALIGN] = apply_r_riscv_align_rela, + [R_RISCV_ADD16] = apply_r_riscv_add16_rela, [R_RISCV_ADD32] = apply_r_riscv_add32_rela, [R_RISCV_ADD64] = apply_r_riscv_add64_rela, + [R_RISCV_SUB16] = apply_r_riscv_sub16_rela, [R_RISCV_SUB32] = apply_r_riscv_sub32_rela, [R_RISCV_SUB64] = apply_r_riscv_sub64_rela, }; From 8d23e94a443388e81c42ea7e476a5d79c1c795c9 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:52 +0800 Subject: [PATCH 31/67] riscv: switch to relative alternative entries Instead of using absolute addresses for both the old instrucions and the alternative instructions, use offsets relative to the alt_entry values. So this not only cuts the size of the alternative entry, but also meets the prerequisite for patching alternatives in the vDSO, since absolute alternative entries are subject to dynamic relocation, which is incompatible with the vDSO building. Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-10-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/errata/sifive/errata.c | 3 ++- arch/riscv/errata/thead/errata.c | 11 ++++++++--- arch/riscv/include/asm/alternative-macros.h | 20 ++++++++++---------- arch/riscv/include/asm/alternative.h | 17 +++++++++++------ arch/riscv/kernel/cpufeature.c | 8 +++++--- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index 1031038423e7..ef9a4eec0dba 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -107,7 +107,8 @@ void __init_or_module sifive_errata_patch_func(struct alt_entry *begin, tmp = (1U << alt->errata_id); if (cpu_req_errata & tmp) { - patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); + patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt), + alt->alt_len); cpu_apply_errata |= tmp; } } diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c index fac5742d1c1e..c0bea5c94128 100644 --- a/arch/riscv/errata/thead/errata.c +++ b/arch/riscv/errata/thead/errata.c @@ -87,6 +87,7 @@ void __init_or_module thead_errata_patch_func(struct alt_entry *begin, struct al struct alt_entry *alt; u32 cpu_req_errata = thead_errata_probe(stage, archid, impid); u32 tmp; + void *oldptr, *altptr; for (alt = begin; alt < end; alt++) { if (alt->vendor_id != THEAD_VENDOR_ID) @@ -96,12 +97,16 @@ void __init_or_module thead_errata_patch_func(struct alt_entry *begin, struct al tmp = (1U << alt->errata_id); if (cpu_req_errata & tmp) { + oldptr = ALT_OLD_PTR(alt); + altptr = ALT_ALT_PTR(alt); + /* On vm-alternatives, the mmu isn't running yet */ if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) - memcpy((void *)__pa_symbol(alt->old_ptr), - (void *)__pa_symbol(alt->alt_ptr), alt->alt_len); + memcpy((void *)__pa_symbol(oldptr), + (void *)__pa_symbol(altptr), + alt->alt_len); else - patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); + patch_text_nosync(oldptr, altptr, alt->alt_len); } } diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h index 7226e2462584..cc6a81c00f2f 100644 --- a/arch/riscv/include/asm/alternative-macros.h +++ b/arch/riscv/include/asm/alternative-macros.h @@ -7,11 +7,11 @@ #ifdef __ASSEMBLY__ .macro ALT_ENTRY oldptr newptr vendor_id errata_id new_len - RISCV_PTR \oldptr - RISCV_PTR \newptr - REG_ASM \vendor_id - REG_ASM \new_len - .word \errata_id + .4byte \oldptr - . + .4byte \newptr - . + .2byte \vendor_id + .2byte \new_len + .4byte \errata_id .endm .macro ALT_NEW_CONTENT vendor_id, errata_id, enable = 1, new_c : vararg @@ -59,11 +59,11 @@ #include #define ALT_ENTRY(oldptr, newptr, vendor_id, errata_id, newlen) \ - RISCV_PTR " " oldptr "\n" \ - RISCV_PTR " " newptr "\n" \ - REG_ASM " " vendor_id "\n" \ - REG_ASM " " newlen "\n" \ - ".word " errata_id "\n" + ".4byte ((" oldptr ") - .) \n" \ + ".4byte ((" newptr ") - .) \n" \ + ".2byte " vendor_id "\n" \ + ".2byte " newlen "\n" \ + ".4byte " errata_id "\n" #define ALT_NEW_CONTENT(vendor_id, errata_id, enable, new_c) \ ".if " __stringify(enable) " == 1\n" \ diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h index 1bd4027d34ca..b8648d4f2ac1 100644 --- a/arch/riscv/include/asm/alternative.h +++ b/arch/riscv/include/asm/alternative.h @@ -23,6 +23,11 @@ #define RISCV_ALTERNATIVES_MODULE 1 /* alternatives applied during module-init */ #define RISCV_ALTERNATIVES_EARLY_BOOT 2 /* alternatives applied before mmu start */ +/* add the relative offset to the address of the offset to get the absolute address */ +#define __ALT_PTR(a, f) ((void *)&(a)->f + (a)->f) +#define ALT_OLD_PTR(a) __ALT_PTR(a, old_offset) +#define ALT_ALT_PTR(a) __ALT_PTR(a, alt_offset) + void __init apply_boot_alternatives(void); void __init apply_early_boot_alternatives(void); void apply_module_alternatives(void *start, size_t length); @@ -31,12 +36,12 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, int patch_offset); struct alt_entry { - void *old_ptr; /* address of original instruciton or data */ - void *alt_ptr; /* address of replacement instruction or data */ - unsigned long vendor_id; /* cpu vendor id */ - unsigned long alt_len; /* The replacement size */ - unsigned int errata_id; /* The errata id */ -} __packed; + s32 old_offset; /* offset relative to original instruction or data */ + s32 alt_offset; /* offset relative to replacement instruction or data */ + u16 vendor_id; /* cpu vendor id */ + u16 alt_len; /* The replacement size */ + u32 errata_id; /* The errata id */ +}; struct errata_checkfunc_id { unsigned long vendor_id; diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 6193f401f0c5..18ea518f9e68 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -281,6 +281,7 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, unsigned int stage) { struct alt_entry *alt; + void *oldptr, *altptr; if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) return; @@ -297,9 +298,10 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, if (!__riscv_isa_extension_available(NULL, alt->errata_id)) continue; - patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); - riscv_alternative_fix_offsets(alt->old_ptr, alt->alt_len, - alt->old_ptr - alt->alt_ptr); + oldptr = ALT_OLD_PTR(alt); + altptr = ALT_ALT_PTR(alt); + patch_text_nosync(oldptr, altptr, alt->alt_len); + riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); } } #endif From cabfd146b371462ed480a968f98eb0c2c80e3c8a Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:53 +0800 Subject: [PATCH 32/67] riscv: alternative: patch alternatives in the vDSO Make it possible to use alternatives in the vDSO, so that better implementations can be used if possible. Signed-off-by: Jisheng Zhang Reviewed-by: Guo Ren Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20230128172856.3814-11-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/vdso.h | 4 ++++ arch/riscv/kernel/alternative.c | 29 +++++++++++++++++++++++++++++ arch/riscv/kernel/vdso.c | 5 ----- arch/riscv/kernel/vdso/vdso.lds.S | 7 +++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index a7644f46d0e5..f891478829a5 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -28,8 +28,12 @@ #define COMPAT_VDSO_SYMBOL(base, name) \ (void __user *)((unsigned long)(base) + compat__vdso_##name##_offset) +extern char compat_vdso_start[], compat_vdso_end[]; + #endif /* CONFIG_COMPAT */ +extern char vdso_start[], vdso_end[]; + #endif /* !__ASSEMBLY__ */ #endif /* CONFIG_MMU */ diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 3d4f1f32c7f6..fc341b69bf62 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -160,6 +162,31 @@ static void __init_or_module _apply_alternatives(struct alt_entry *begin, stage); } +#ifdef CONFIG_MMU +static void __init apply_vdso_alternatives(void) +{ + const Elf_Ehdr *hdr; + const Elf_Shdr *shdr; + const Elf_Shdr *alt; + struct alt_entry *begin, *end; + + hdr = (Elf_Ehdr *)vdso_start; + shdr = (void *)hdr + hdr->e_shoff; + alt = find_section(hdr, shdr, ".alternative"); + if (!alt) + return; + + begin = (void *)hdr + alt->sh_offset, + end = (void *)hdr + alt->sh_offset + alt->sh_size, + + _apply_alternatives((struct alt_entry *)begin, + (struct alt_entry *)end, + RISCV_ALTERNATIVES_BOOT); +} +#else +static void __init apply_vdso_alternatives(void) { } +#endif + void __init apply_boot_alternatives(void) { /* If called on non-boot cpu things could go wrong */ @@ -168,6 +195,8 @@ void __init apply_boot_alternatives(void) _apply_alternatives((struct alt_entry *)__alt_start, (struct alt_entry *)__alt_end, RISCV_ALTERNATIVES_BOOT); + + apply_vdso_alternatives(); } /* diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index e410275918ac..4e631c098f4d 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -22,11 +22,6 @@ struct vdso_data { }; #endif -extern char vdso_start[], vdso_end[]; -#ifdef CONFIG_COMPAT -extern char compat_vdso_start[], compat_vdso_end[]; -#endif - enum vvar_pages { VVAR_DATA_PAGE_OFFSET, VVAR_TIMENS_PAGE_OFFSET, diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index 150b1a572e61..4a0606633290 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -40,6 +40,13 @@ SECTIONS . = 0x800; .text : { *(.text .text.*) } :text + . = ALIGN(4); + .alternative : { + __alt_start = .; + *(.alternative) + __alt_end = .; + } + .data : { *(.got.plt) *(.got) *(.data .data.* .gnu.linkonce.d.*) From 95bc69a47be2d5cdccf40ba3f23c99e9a6c57597 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:54 +0800 Subject: [PATCH 33/67] riscv: cpu_relax: switch to riscv_has_extension_likely() Switch cpu_relax() from static branch to the new helper riscv_has_extension_likely() Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Reviewed-by: Heiko Stuebner Reviewed-by: Guo Ren Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230128172856.3814-12-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/vdso/processor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h index fa70cfe507aa..edf0e25e43d1 100644 --- a/arch/riscv/include/asm/vdso/processor.h +++ b/arch/riscv/include/asm/vdso/processor.h @@ -10,7 +10,7 @@ static inline void cpu_relax(void) { - if (!static_branch_likely(&riscv_isa_ext_keys[RISCV_ISA_EXT_KEY_ZIHINTPAUSE])) { + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZIHINTPAUSE)) { #ifdef __riscv_muldiv int dummy; /* In lieu of a halt instruction, induce a long-latency stall. */ From e8ad17d2b5f38e595d597a3e2419d6d7cc727b17 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Sun, 29 Jan 2023 01:28:55 +0800 Subject: [PATCH 34/67] riscv: KVM: Switch has_svinval() to riscv_has_extension_unlikely() Switch has_svinval() from static branch to the new helper riscv_has_extension_unlikely(). Signed-off-by: Andrew Jones Reviewed-by: Guo Ren Acked-by: Anup Patel Link: https://lore.kernel.org/r/20230128172856.3814-13-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kvm/tlb.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index 309d79b3e5cd..aa3da18ad873 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -15,8 +15,7 @@ #include #include -#define has_svinval() \ - static_branch_unlikely(&riscv_isa_ext_keys[RISCV_ISA_EXT_KEY_SVINVAL]) +#define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, gpa_t gpa, gpa_t gpsz, From 03966594e1170303c037b0cded35c464a13a4a45 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sun, 29 Jan 2023 01:28:56 +0800 Subject: [PATCH 35/67] riscv: remove riscv_isa_ext_keys[] array and related usage All users have switched to riscv_has_extension_*, remove unused definitions, vars and related setting code. Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Reviewed-by: Heiko Stuebner Reviewed-by: Conor Dooley Reviewed-by: Guo Ren Link: https://lore.kernel.org/r/20230128172856.3814-14-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 32 -------------------------------- arch/riscv/kernel/cpufeature.c | 9 --------- 2 files changed, 41 deletions(-) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 411ef0fb5c4b..7936ae6f7bdf 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -61,20 +61,6 @@ enum { extern unsigned long elf_hwcap; - -/* - * This enum represents the logical ID for each RISC-V ISA extension static - * keys. We can use static key to optimize code path if some ISA extensions - * are available. - * Entries are sorted alphabetically. - */ -enum riscv_isa_ext_key { - RISCV_ISA_EXT_KEY_FPU, /* For 'F' and 'D' */ - RISCV_ISA_EXT_KEY_SVINVAL, - RISCV_ISA_EXT_KEY_ZIHINTPAUSE, - RISCV_ISA_EXT_KEY_MAX, -}; - struct riscv_isa_ext_data { /* Name of the extension displayed to userspace via /proc/cpuinfo */ char uprop[RISCV_ISA_EXT_NAME_LEN_MAX]; @@ -82,24 +68,6 @@ struct riscv_isa_ext_data { unsigned int isa_ext_id; }; -extern struct static_key_false riscv_isa_ext_keys[RISCV_ISA_EXT_KEY_MAX]; - -static __always_inline int riscv_isa_ext2key(int num) -{ - switch (num) { - case RISCV_ISA_EXT_f: - return RISCV_ISA_EXT_KEY_FPU; - case RISCV_ISA_EXT_d: - return RISCV_ISA_EXT_KEY_FPU; - case RISCV_ISA_EXT_SVINVAL: - return RISCV_ISA_EXT_KEY_SVINVAL; - case RISCV_ISA_EXT_ZIHINTPAUSE: - return RISCV_ISA_EXT_KEY_ZIHINTPAUSE; - default: - return -EINVAL; - } -} - static __always_inline bool riscv_has_extension_likely(const unsigned long ext) { diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 18ea518f9e68..a4f737bc7530 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -29,9 +29,6 @@ unsigned long elf_hwcap __read_mostly; /* Host ISA bitmap */ static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; -DEFINE_STATIC_KEY_ARRAY_FALSE(riscv_isa_ext_keys, RISCV_ISA_EXT_KEY_MAX); -EXPORT_SYMBOL(riscv_isa_ext_keys); - /** * riscv_isa_extension_base() - Get base extension word * @@ -267,12 +264,6 @@ void __init riscv_fill_hwcap(void) if (elf_hwcap & BIT_MASK(i)) print_str[j++] = (char)('a' + i); pr_info("riscv: ELF capabilities %s\n", print_str); - - for_each_set_bit(i, riscv_isa, RISCV_ISA_EXT_MAX) { - j = riscv_isa_ext2key(i); - if (j >= 0) - static_branch_enable(&riscv_isa_ext_keys[j]); - } } #ifdef CONFIG_RISCV_ALTERNATIVE From 2a5303b499b18de7179ee1b4ab759880fb02ec9c Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Sun, 29 Jan 2023 23:57:01 +0000 Subject: [PATCH 36/67] Documentation: riscv: fix insufficient list item indent When adding the ISA string ordering rules, I didn't sufficiently indent one of the list items. Reported-by: kernel test robot Link: https://lore.kernel.org/linux-doc/202301300743.bp7Dpazv-lkp@intel.com/ Fixes: f07b2b3f9d47 ("Documentation: riscv: add a section about ISA string ordering in /proc/cpuinfo") Signed-off-by: Conor Dooley Reviewed-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20230129235701.2393241-1-conor@kernel.org Signed-off-by: Palmer Dabbelt --- Documentation/riscv/uabi.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/riscv/uabi.rst b/Documentation/riscv/uabi.rst index 2ebec4c52230..8960fac42c40 100644 --- a/Documentation/riscv/uabi.rst +++ b/Documentation/riscv/uabi.rst @@ -21,10 +21,10 @@ so for our purposes the following rules apply: single-letter extensions and before any higher-privileged extensions. #. For additional standard extensions, the first letter following the 'Z' - conventionally indicates the most closely related alphabetical - extension category. If multiple 'Z' extensions are named, they will be ordered - first by category, in canonical order, as listed above, then alphabetically - within a category. + conventionally indicates the most closely related alphabetical + extension category. If multiple 'Z' extensions are named, they will be + ordered first by category, in canonical order, as listed above, then + alphabetically within a category. #. Standard supervisor-level extensions (starting with 'S') will be listed after standard unprivileged extensions. If multiple supervisor-level From 5f365c133b83a584953179fc771cf131668e82ee Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Mon, 19 Dec 2022 17:28:37 +0000 Subject: [PATCH 37/67] RISC-V: re-order Kconfig selects alphanumerically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selects should be sorted alphanumerically, and were tidied up originally by Palmer in commit e8c7ef7d5819 ("RISC-V: Sort select statements alphanumerically") since then, things have gotten out of order again. Fish RMK's original script out of commit b1b3f49ce460 ("ARM: config: sort select statements alphanumerically") and do some spring cleaning. Signed-off-by: Conor Dooley Acked-by: Björn Töpel Link: https://lore.kernel.org/r/20221219172836.134709-1-conor@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 52 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7c814fbf9527..cebf0c5f8824 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -15,10 +15,11 @@ config RISCV select ARCH_CLOCKSOURCE_INIT select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 + select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CURRENT_STACK_POINTER - select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_VIRTUAL if MMU + select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_WX select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_GCOV_PROFILE_ALL @@ -49,8 +50,8 @@ config RISCV select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU select BUILDTIME_TABLE_SORT if MMU - select CLONE_BACKWARDS select CLINT_TIMER if !MMU + select CLONE_BACKWARDS select COMMON_CLK select CPU_PM if CPU_IDLE select EDAC_SUPPORT @@ -85,16 +86,16 @@ config RISCV select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_THREAD_STRUCT_WHITELIST select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if 64BIT && MMU - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE - select HAVE_ARCH_THREAD_STRUCT_WHITELIST select HAVE_ARCH_VMAP_STACK if MMU && 64BIT select HAVE_ASM_MODVERSIONS select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU + select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_ERROR_INJECTION select HAVE_GCC_PLUGINS select HAVE_GENERIC_VDSO if MMU && 64BIT @@ -111,10 +112,9 @@ config RISCV select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_REGS_AND_STACK_ACCESS_API - select HAVE_FUNCTION_ARG_ACCESS_API + select HAVE_RSEQ select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS - select HAVE_RSEQ select IRQ_DOMAIN select IRQ_FORCED_THREADING select MODULES_USE_ELF_RELA if MODULES @@ -235,9 +235,9 @@ config LOCKDEP_SUPPORT config RISCV_DMA_NONCOHERENT bool select ARCH_HAS_DMA_PREP_COHERENT - select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SETUP_DMA_OPS + select ARCH_HAS_SYNC_DMA_FOR_CPU + select ARCH_HAS_SYNC_DMA_FOR_DEVICE select DMA_DIRECT_REMAP config AS_HAS_INSN @@ -352,11 +352,11 @@ endchoice config NUMA bool "NUMA Memory Allocation and Scheduler Support" depends on SMP && MMU - select GENERIC_ARCH_NUMA - select OF_NUMA select ARCH_SUPPORTS_NUMA_BALANCING - select USE_PERCPU_NUMA_NODE_ID + select GENERIC_ARCH_NUMA select NEED_PER_CPU_EMBED_FIRST_CHUNK + select OF_NUMA + select USE_PERCPU_NUMA_NODE_ID help Enable NUMA (Non-Uniform Memory Access) support. @@ -401,8 +401,8 @@ config RISCV_ISA_SVPBMT bool "SVPBMT extension support" depends on 64BIT && MMU depends on !XIP_KERNEL - select RISCV_ALTERNATIVE default y + select RISCV_ALTERNATIVE help Adds support to dynamically detect the presence of the SVPBMT ISA-extension (Supervisor-mode: page-based memory types) and @@ -451,9 +451,9 @@ config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on TOOLCHAIN_HAS_ZICBOM depends on !XIP_KERNEL && MMU - select RISCV_DMA_NONCOHERENT - select RISCV_ALTERNATIVE default y + select RISCV_ALTERNATIVE + select RISCV_DMA_NONCOHERENT help Adds support to dynamically detect the presence of the ZICBOM extension (Cache Block Management Operations) and enable its @@ -515,9 +515,9 @@ config RISCV_BOOT_SPINWAIT config KEXEC bool "Kexec system call" - select KEXEC_CORE - select HOTPLUG_CPU if SMP depends on MMU + select HOTPLUG_CPU if SMP + select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot @@ -528,10 +528,10 @@ config KEXEC config KEXEC_FILE bool "kexec file based systmem call" + depends on 64BIT && MMU + select HAVE_IMA_KEXEC if IMA select KEXEC_CORE select KEXEC_ELF - select HAVE_IMA_KEXEC if IMA - depends on 64BIT && MMU help This is new version of kexec system call. This system call is file based and takes file descriptors as system call argument @@ -620,15 +620,15 @@ config EFI_STUB config EFI bool "UEFI runtime support" depends on OF && !XIP_KERNEL - select LIBFDT - select UCS2_STRING - select EFI_PARAMS_FROM_FDT - select EFI_STUB - select EFI_GENERIC_STUB - select EFI_RUNTIME_WRAPPERS - select RISCV_ISA_C depends on MMU default y + select EFI_GENERIC_STUB + select EFI_PARAMS_FROM_FDT + select EFI_RUNTIME_WRAPPERS + select EFI_STUB + select LIBFDT + select RISCV_ISA_C + select UCS2_STRING help This option provides support for runtime services provided by UEFI firmware (such as non-volatile variables, realtime @@ -707,8 +707,8 @@ config PORTABLE bool default !NONPORTABLE select EFI - select OF select MMU + select OF menu "Power management options" From bfd6fc5d80145e12d0ffa144c4bad89b8f9ddc5a Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 11 Feb 2023 20:15:32 -0600 Subject: [PATCH 38/67] riscv: Fix early alternative patching Now that the text to patch is located using a relative offset from the alternative entry, the text address should be computed without applying the kernel mapping offset, both before and after VM setup. Fixes: 8d23e94a4433 ("riscv: switch to relative alternative entries") Signed-off-by: Samuel Holland Reviewed-by: Conor Dooley Reviewed-by: Guo Ren Reviewed-by: Jisheng Zhang Tested-by: Conor Dooley Link: https://lore.kernel.org/r/20230212021534.59121-2-samuel@sholland.org Signed-off-by: Palmer Dabbelt --- arch/riscv/errata/thead/errata.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c index c0bea5c94128..1dd90a5f86f0 100644 --- a/arch/riscv/errata/thead/errata.c +++ b/arch/riscv/errata/thead/errata.c @@ -102,9 +102,7 @@ void __init_or_module thead_errata_patch_func(struct alt_entry *begin, struct al /* On vm-alternatives, the mmu isn't running yet */ if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) - memcpy((void *)__pa_symbol(oldptr), - (void *)__pa_symbol(altptr), - alt->alt_len); + memcpy(oldptr, altptr, alt->alt_len); else patch_text_nosync(oldptr, altptr, alt->alt_len); } From d5a7fab7859dc88657372a448b78babcf134114e Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Sat, 11 Feb 2023 20:15:33 -0600 Subject: [PATCH 39/67] riscv: Fix Zbb alternative IDs Commit 4bf8860760d9 ("riscv: cpufeature: extend riscv_cpufeature_patch_func to all ISA extensions") switched ISA extension alternatives to use the RISCV_ISA_EXT_* macros instead of CPUFEATURE_*. This was mismerged when applied on top of the Zbb series, so the Zbb alternatives referenced the wrong errata ID values. Fixes: 9daca9a5b9ac ("Merge patch series "riscv: improve boot time isa extensions handling"") Signed-off-by: Samuel Holland Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Reviewed-by: Guo Ren Tested-by: Conor Dooley Link: https://lore.kernel.org/r/20230212021534.59121-3-samuel@sholland.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/errata_list.h | 5 ----- arch/riscv/lib/strcmp.S | 2 +- arch/riscv/lib/strlen.S | 2 +- arch/riscv/lib/strncmp.S | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index e158439029ce..274c6f889602 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -23,11 +23,6 @@ #define ERRATA_THEAD_NUMBER 3 #endif -#define CPUFEATURE_SVPBMT 0 -#define CPUFEATURE_ZICBOM 1 -#define CPUFEATURE_ZBB 2 -#define CPUFEATURE_NUMBER 3 - #ifdef __ASSEMBLY__ #define ALT_INSN_FAULT(x) \ diff --git a/arch/riscv/lib/strcmp.S b/arch/riscv/lib/strcmp.S index 8148b6418f61..986ab23fe787 100644 --- a/arch/riscv/lib/strcmp.S +++ b/arch/riscv/lib/strcmp.S @@ -9,7 +9,7 @@ /* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) - ALTERNATIVE("nop", "j strcmp_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + ALTERNATIVE("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) /* * Returns diff --git a/arch/riscv/lib/strlen.S b/arch/riscv/lib/strlen.S index 0f9dbf93301a..8345ceeee3f6 100644 --- a/arch/riscv/lib/strlen.S +++ b/arch/riscv/lib/strlen.S @@ -9,7 +9,7 @@ /* int strlen(const char *s) */ SYM_FUNC_START(strlen) - ALTERNATIVE("nop", "j strlen_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + ALTERNATIVE("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) /* * Returns diff --git a/arch/riscv/lib/strncmp.S b/arch/riscv/lib/strncmp.S index 7940ddab2d48..ee49595075be 100644 --- a/arch/riscv/lib/strncmp.S +++ b/arch/riscv/lib/strncmp.S @@ -9,7 +9,7 @@ /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) - ALTERNATIVE("nop", "j strncmp_zbb", 0, CPUFEATURE_ZBB, CONFIG_RISCV_ISA_ZBB) + ALTERNATIVE("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) /* * Returns From 7d2078310cbf0fa7fb4323d595fe093c418dcd37 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 4 Jan 2023 18:05:13 +0000 Subject: [PATCH 40/67] dt-bindings: arm: move cpu-capacity to a shared loation RISC-V uses the same generic topology code as arm64 & while there currently exists no binding for cpu-capacity on RISC-V, the code paths can be hit if the property is present. Move the documentation of cpu-capacity to a shared location, ahead of defining a binding for capacity-dmips-mhz on RISC-V. Update some references to this document in the process. Signed-off-by: Conor Dooley Reviewed-by: Ley Foon Tan Acked-by: Rob Herring Reviewed-by: Yanteng Si Link: https://lore.kernel.org/r/20230104180513.1379453-2-conor@kernel.org Signed-off-by: Palmer Dabbelt --- Documentation/devicetree/bindings/arm/cpus.yaml | 2 +- .../devicetree/bindings/{arm => cpu}/cpu-capacity.txt | 4 ++-- Documentation/scheduler/sched-capacity.rst | 2 +- Documentation/translations/zh_CN/scheduler/sched-capacity.rst | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename Documentation/devicetree/bindings/{arm => cpu}/cpu-capacity.txt (98%) diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml index 01b5a9c689a2..a7586295a6f5 100644 --- a/Documentation/devicetree/bindings/arm/cpus.yaml +++ b/Documentation/devicetree/bindings/arm/cpus.yaml @@ -257,7 +257,7 @@ properties: capacity-dmips-mhz: description: - u32 value representing CPU capacity (see ./cpu-capacity.txt) in + u32 value representing CPU capacity (see ../cpu/cpu-capacity.txt) in DMIPS/MHz, relative to highest capacity-dmips-mhz in the system. diff --git a/Documentation/devicetree/bindings/arm/cpu-capacity.txt b/Documentation/devicetree/bindings/cpu/cpu-capacity.txt similarity index 98% rename from Documentation/devicetree/bindings/arm/cpu-capacity.txt rename to Documentation/devicetree/bindings/cpu/cpu-capacity.txt index cc5e190390b7..f28e1adad428 100644 --- a/Documentation/devicetree/bindings/arm/cpu-capacity.txt +++ b/Documentation/devicetree/bindings/cpu/cpu-capacity.txt @@ -1,12 +1,12 @@ ========================================== -ARM CPUs capacity bindings +CPU capacity bindings ========================================== ========================================== 1 - Introduction ========================================== -ARM systems may be configured to have cpus with different power/performance +Some systems may be configured to have cpus with different power/performance characteristics within the same chip. In this case, additional information has to be made available to the kernel for it to be aware of such differences and take decisions accordingly. diff --git a/Documentation/scheduler/sched-capacity.rst b/Documentation/scheduler/sched-capacity.rst index 805f85f330b5..8e2b8538bc2b 100644 --- a/Documentation/scheduler/sched-capacity.rst +++ b/Documentation/scheduler/sched-capacity.rst @@ -260,7 +260,7 @@ for that purpose. The arm and arm64 architectures directly map this to the arch_topology driver CPU scaling data, which is derived from the capacity-dmips-mhz CPU binding; see -Documentation/devicetree/bindings/arm/cpu-capacity.txt. +Documentation/devicetree/bindings/cpu/cpu-capacity.txt. 3.2 Frequency invariance ------------------------ diff --git a/Documentation/translations/zh_CN/scheduler/sched-capacity.rst b/Documentation/translations/zh_CN/scheduler/sched-capacity.rst index 3a52053c29dc..e07ffdd391d3 100644 --- a/Documentation/translations/zh_CN/scheduler/sched-capacity.rst +++ b/Documentation/translations/zh_CN/scheduler/sched-capacity.rst @@ -233,7 +233,7 @@ CFS调度类基于实体负载跟踪机制（Per-Entity Load Tracking, PELT） arm和arm64架构直接把这个信息映射到arch_topology驱动的CPU scaling数据中（译注：参考 arch_topology.h的percpu变量cpu_scale），它是从capacity-dmips-mhz CPU binding中衍生计算 -出来的。参见Documentation/devicetree/bindings/arm/cpu-capacity.txt。 +出来的。参见Documentation/devicetree/bindings/cpu/cpu-capacity.txt。 3.2 频率不变性 -------------- From 991994509ee93f7698251e696b8e5591e01b7f68 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 4 Jan 2023 18:05:14 +0000 Subject: [PATCH 41/67] dt-bindings: riscv: add a capacity-dmips-mhz cpu property Since commit 03f11f03dbfe ("RISC-V: Parse cpu topology during boot.") RISC-V has used the generic arch topology code, which provides for disparate CPU capacities. We never defined a binding to acquire this information from the DT though, so document the one already used by the generic arch topology code: "capacity-dmips-mhz". Signed-off-by: Conor Dooley Reviewed-by: Ley Foon Tan Acked-by: Rob Herring Link: https://lore.kernel.org/r/20230104180513.1379453-3-conor@kernel.org Signed-off-by: Palmer Dabbelt --- Documentation/devicetree/bindings/riscv/cpus.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml index a2884e3113da..001931d526ec 100644 --- a/Documentation/devicetree/bindings/riscv/cpus.yaml +++ b/Documentation/devicetree/bindings/riscv/cpus.yaml @@ -114,6 +114,12 @@ properties: List of phandles to idle state nodes supported by this hart (see ./idle-states.yaml). + capacity-dmips-mhz: + description: + u32 value representing CPU capacity (see ../cpu/cpu-capacity.txt) in + DMIPS/MHz, relative to highest capacity-dmips-mhz + in the system. + required: - riscv,isa - interrupt-controller From 6067c3aae52f14b025d0a4de5d4d1eb16eb8fe51 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Sun, 8 Jan 2023 16:33:54 +0000 Subject: [PATCH 42/67] RISC-V: insn-def: Add I-type insn-def CBO instructions use the I-type of instruction format where the immediate is used to identify the CBO instruction type. Add I-type instruction encoding support to insn-def. Signed-off-by: Andrew Jones Reviewed-by: Conor Dooley Reviewed-by: Heiko Stuebner Signed-off-by: Conor Dooley Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20230108163356.3063839-2-conor@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/insn-def.h | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index 16044affa57c..39f9e0893564 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -12,6 +12,12 @@ #define INSN_R_RD_SHIFT 7 #define INSN_R_OPCODE_SHIFT 0 +#define INSN_I_SIMM12_SHIFT 20 +#define INSN_I_RS1_SHIFT 15 +#define INSN_I_FUNC3_SHIFT 12 +#define INSN_I_RD_SHIFT 7 +#define INSN_I_OPCODE_SHIFT 0 + #ifdef __ASSEMBLY__ #ifdef CONFIG_AS_HAS_INSN @@ -20,6 +26,10 @@ .insn r \opcode, \func3, \func7, \rd, \rs1, \rs2 .endm + .macro insn_i, opcode, func3, rd, rs1, simm12 + .insn i \opcode, \func3, \rd, \rs1, \simm12 + .endm + #else #include @@ -33,9 +43,18 @@ (.L__gpr_num_\rs2 << INSN_R_RS2_SHIFT)) .endm + .macro insn_i, opcode, func3, rd, rs1, simm12 + .4byte ((\opcode << INSN_I_OPCODE_SHIFT) | \ + (\func3 << INSN_I_FUNC3_SHIFT) | \ + (.L__gpr_num_\rd << INSN_I_RD_SHIFT) | \ + (.L__gpr_num_\rs1 << INSN_I_RS1_SHIFT) | \ + (\simm12 << INSN_I_SIMM12_SHIFT)) + .endm + #endif #define __INSN_R(...) insn_r __VA_ARGS__ +#define __INSN_I(...) insn_i __VA_ARGS__ #else /* ! __ASSEMBLY__ */ @@ -44,6 +63,9 @@ #define __INSN_R(opcode, func3, func7, rd, rs1, rs2) \ ".insn r " opcode ", " func3 ", " func7 ", " rd ", " rs1 ", " rs2 "\n" +#define __INSN_I(opcode, func3, rd, rs1, simm12) \ + ".insn i " opcode ", " func3 ", " rd ", " rs1 ", " simm12 "\n" + #else #include @@ -60,14 +82,32 @@ " (.L__gpr_num_\\rs2 << " __stringify(INSN_R_RS2_SHIFT) "))\n" \ " .endm\n" +#define DEFINE_INSN_I \ + __DEFINE_ASM_GPR_NUMS \ +" .macro insn_i, opcode, func3, rd, rs1, simm12\n" \ +" .4byte ((\\opcode << " __stringify(INSN_I_OPCODE_SHIFT) ") |" \ +" (\\func3 << " __stringify(INSN_I_FUNC3_SHIFT) ") |" \ +" (.L__gpr_num_\\rd << " __stringify(INSN_I_RD_SHIFT) ") |" \ +" (.L__gpr_num_\\rs1 << " __stringify(INSN_I_RS1_SHIFT) ") |" \ +" (\\simm12 << " __stringify(INSN_I_SIMM12_SHIFT) "))\n" \ +" .endm\n" + #define UNDEFINE_INSN_R \ " .purgem insn_r\n" +#define UNDEFINE_INSN_I \ +" .purgem insn_i\n" + #define __INSN_R(opcode, func3, func7, rd, rs1, rs2) \ DEFINE_INSN_R \ "insn_r " opcode ", " func3 ", " func7 ", " rd ", " rs1 ", " rs2 "\n" \ UNDEFINE_INSN_R +#define __INSN_I(opcode, func3, rd, rs1, simm12) \ + DEFINE_INSN_I \ + "insn_i " opcode ", " func3 ", " rd ", " rs1 ", " simm12 "\n" \ + UNDEFINE_INSN_I + #endif #endif /* ! __ASSEMBLY__ */ @@ -76,9 +116,14 @@ __INSN_R(RV_##opcode, RV_##func3, RV_##func7, \ RV_##rd, RV_##rs1, RV_##rs2) +#define INSN_I(opcode, func3, rd, rs1, simm12) \ + __INSN_I(RV_##opcode, RV_##func3, RV_##rd, \ + RV_##rs1, RV_##simm12) + #define RV_OPCODE(v) __ASM_STR(v) #define RV_FUNC3(v) __ASM_STR(v) #define RV_FUNC7(v) __ASM_STR(v) +#define RV_SIMM12(v) __ASM_STR(v) #define RV_RD(v) __ASM_STR(v) #define RV_RS1(v) __ASM_STR(v) #define RV_RS2(v) __ASM_STR(v) @@ -87,6 +132,7 @@ #define RV___RS1(v) __RV_REG(v) #define RV___RS2(v) __RV_REG(v) +#define RV_OPCODE_MISC_MEM RV_OPCODE(15) #define RV_OPCODE_SYSTEM RV_OPCODE(115) #define HFENCE_VVMA(vaddr, asid) \ From dd23e9535889c8b8d7ea7d6064ad16be903f4270 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Sun, 8 Jan 2023 16:33:55 +0000 Subject: [PATCH 43/67] RISC-V: replace cbom instructions with an insn-def Using the cbom instructions directly in ALT_CMO_OP, requires toolchain support for the instructions. Using an insn-def will allow for removal of toolchain version checks in the build system & simplification of the proposed [1] function-based CMO scheme. Link: https://lore.kernel.org/linux-riscv/fb3b34ae-e35e-4dc2-a8f4-19984a2f58a8@app.fastmail.com/ [1] Signed-off-by: Conor Dooley Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20230108163356.3063839-3-conor@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/errata_list.h | 3 ++- arch/riscv/include/asm/insn-def.h | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index 4180312d2a70..b71e3ccd3110 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -7,6 +7,7 @@ #include #include +#include #include #ifdef CONFIG_ERRATA_SIFIVE @@ -125,7 +126,7 @@ asm volatile(ALTERNATIVE_2( \ "mv a0, %1\n\t" \ "j 2f\n\t" \ "3:\n\t" \ - "cbo." __stringify(_op) " (a0)\n\t" \ + CBO_##_op(a0) \ "add a0, a0, %0\n\t" \ "2:\n\t" \ "bltu a0, %2, 3b\n\t" \ diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index 39f9e0893564..e01ab51f50d2 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -180,4 +180,16 @@ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(51), \ __RD(0), RS1(gaddr), RS2(vmid)) +#define CBO_inval(base) \ + INSN_I(OPCODE_MISC_MEM, FUNC3(2), __RD(0), \ + RS1(base), SIMM12(0)) + +#define CBO_clean(base) \ + INSN_I(OPCODE_MISC_MEM, FUNC3(2), __RD(0), \ + RS1(base), SIMM12(1)) + +#define CBO_flush(base) \ + INSN_I(OPCODE_MISC_MEM, FUNC3(2), __RD(0), \ + RS1(base), SIMM12(2)) + #endif /* __ASM_INSN_DEF_H */ From 75c53905f8738fecf691a78de401250e298900f9 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Sun, 8 Jan 2023 16:33:56 +0000 Subject: [PATCH 44/67] RISC-V: remove toolchain version checks for Zicbom Commit b8c86872d1dc ("riscv: fix detection of toolchain Zicbom support") fixed building on systems where Zicbom was supported by the compiler/assembler but not by the linker in an easily backportable manner. Now that the we have insn-defs for the 3 instructions, toolchain support is no longer required for Zicbom. Stop emitting "_zicbom" in -march when Zicbom is enabled & drop the version checks entirely. Signed-off-by: Conor Dooley Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20230108163356.3063839-4-conor@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 8 -------- arch/riscv/Makefile | 3 --- 2 files changed, 11 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index e2b656043abf..33bbdc33cef8 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -416,16 +416,8 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. -config TOOLCHAIN_HAS_ZICBOM - bool - default y - depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zicbom) - depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zicbom) - depends on LLD_VERSION >= 150000 || LD_VERSION >= 23800 - config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" - depends on TOOLCHAIN_HAS_ZICBOM depends on !XIP_KERNEL && MMU select RISCV_DMA_NONCOHERENT select RISCV_ALTERNATIVE diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index faf2c2177094..b5d69e5a5c81 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -58,9 +58,6 @@ riscv-march-$(CONFIG_RISCV_ISA_C) := $(riscv-march-y)c toolchain-need-zicsr-zifencei := $(call cc-option-yn, -march=$(riscv-march-y)_zicsr_zifencei) riscv-march-$(toolchain-need-zicsr-zifencei) := $(riscv-march-y)_zicsr_zifencei -# Check if the toolchain supports Zicbom extension -riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZICBOM) := $(riscv-march-y)_zicbom - # Check if the toolchain supports Zihintpause extension riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE) := $(riscv-march-y)_zihintpause From 8547649981e6631328cd64f583667501ae385531 Mon Sep 17 00:00:00 2001 From: Andy Chiu Date: Thu, 12 Jan 2023 04:05:57 -0500 Subject: [PATCH 45/67] riscv: ftrace: Fixup panic by disabling preemption In RISCV, we must use an AUIPC + JALR pair to encode an immediate, forming a jump that jumps to an address over 4K. This may cause errors if we want to enable kernel preemption and remove dependency from patching code with stop_machine(). For example, if a task was switched out on auipc. And, if we changed the ftrace function before it was switched back, then it would jump to an address that has updated 11:0 bits mixing with previous XLEN:12 part. p: patched area performed by dynamic ftrace ftrace_prologue: p| REG_S ra, -SZREG(sp) p| auipc ra, 0x? ------------> preempted ... change ftrace function ... p| jalr -?(ra) <------------- switched back p| REG_L ra, -SZREG(sp) func: xxx ret Fixes: afc76b8b8011 ("riscv: Using PATCHABLE_FUNCTION_ENTRY instead of MCOUNT") Signed-off-by: Andy Chiu Signed-off-by: Guo Ren Link: https://lore.kernel.org/r/20230112090603.1295340-2-guoren@kernel.org Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index e2b656043abf..ee0d39b26794 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -138,7 +138,7 @@ config RISCV select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FUNCTION_TRACER if !XIP_KERNEL + select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT From 409c8fb20c66df7150e592747412438c04aeb11f Mon Sep 17 00:00:00 2001 From: Guo Ren Date: Thu, 12 Jan 2023 04:05:58 -0500 Subject: [PATCH 46/67] riscv: ftrace: Remove wasted nops for !RISCV_ISA_C When CONFIG_RISCV_ISA_C=n, -fpatchable-function-entry=8 would generate more nops than we expect. Because it treat nop opcode as 0x00000013 instead of 0x0001. Dump of assembler code for function dw_pcie_free_msi: 0xffffffff806fce94 <+0>: sd ra,-8(sp) 0xffffffff806fce98 <+4>: auipc ra,0xff90f 0xffffffff806fce9c <+8>: jalr -684(ra) # 0xffffffff8000bbec 0xffffffff806fcea0 <+12>: ld ra,-8(sp) 0xffffffff806fcea4 <+16>: nop /* wasted */ 0xffffffff806fcea8 <+20>: nop /* wasted */ 0xffffffff806fceac <+24>: nop /* wasted */ 0xffffffff806fceb0 <+28>: nop /* wasted */ 0xffffffff806fceb4 <+0>: addi sp,sp,-48 0xffffffff806fceb8 <+4>: sd s0,32(sp) 0xffffffff806fcebc <+8>: sd s1,24(sp) 0xffffffff806fcec0 <+12>: sd s2,16(sp) 0xffffffff806fcec4 <+16>: sd s3,8(sp) 0xffffffff806fcec8 <+20>: sd ra,40(sp) 0xffffffff806fcecc <+24>: addi s0,sp,48 Signed-off-by: Guo Ren Signed-off-by: Guo Ren Link: https://lore.kernel.org/r/20230112090603.1295340-3-guoren@kernel.org Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index faf2c2177094..e194916cb5a2 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -11,7 +11,11 @@ LDFLAGS_vmlinux := ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY +ifeq ($(CONFIG_RISCV_ISA_C),y) CC_FLAGS_FTRACE := -fpatchable-function-entry=8 +else + CC_FLAGS_FTRACE := -fpatchable-function-entry=4 +endif endif ifeq ($(CONFIG_CMODEL_MEDLOW),y) From 6724a76cff85ee271bbbff42ac527e4643b2ec52 Mon Sep 17 00:00:00 2001 From: Guo Ren Date: Thu, 12 Jan 2023 04:05:59 -0500 Subject: [PATCH 47/67] riscv: ftrace: Reduce the detour code size to half Use a temporary register to reduce the size of detour code from 16 bytes to 8 bytes. The previous implementation is from 'commit afc76b8b8011 ("riscv: Using PATCHABLE_FUNCTION_ENTRY instead of MCOUNT")'. Before the patch: : 0: REG_S ra, -SZREG(sp) 4: auipc ra, ? 8: jalr ?(ra) 12: REG_L ra, -SZREG(sp) (func_boddy) After the patch: : 0: auipc t0, ? 4: jalr t0, ?(t0) (func_boddy) This patch not just reduces the size of detour code, but also fixes an important issue: An Ftrace callback registered with FTRACE_OPS_FL_IPMODIFY flag can actually change the instruction pointer, e.g. to "replace" the given kernel function with a new one, which is needed for livepatching, etc. In this case, the trampoline (ftrace_regs_caller) would not return to but would rather jump to the new function. So, "REG_L ra, -SZREG(sp)" would not run and the original return address would not be restored. The kernel is likely to hang or crash as a result. This can be easily demonstrated if one tries to "replace", say, cmdline_proc_show() with a new function with the same signature using instruction_pointer_set(&fregs->regs, new_func_addr) in the Ftrace callback. Link: https://lore.kernel.org/linux-riscv/20221122075440.1165172-1-suagrfillet@gmail.com/ Link: https://lore.kernel.org/linux-riscv/d7d5730b-ebef-68e5-5046-e763e1ee6164@yadro.com/ Co-developed-by: Song Shuai Signed-off-by: Song Shuai Signed-off-by: Guo Ren Signed-off-by: Guo Ren Cc: Evgenii Shatokhin Reviewed-by: Evgenii Shatokhin Link: https://lore.kernel.org/r/20230112090603.1295340-4-guoren@kernel.org Cc: stable@vger.kernel.org Fixes: 10626c32e382 ("riscv/ftrace: Add basic support") Signed-off-by: Palmer Dabbelt --- arch/riscv/Makefile | 4 +- arch/riscv/include/asm/ftrace.h | 50 +++++++++++++++++++------ arch/riscv/kernel/ftrace.c | 65 ++++++++++----------------------- arch/riscv/kernel/mcount-dyn.S | 42 ++++++++------------- 4 files changed, 75 insertions(+), 86 deletions(-) diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index e194916cb5a2..ab50724cc554 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -12,9 +12,9 @@ ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY ifeq ($(CONFIG_RISCV_ISA_C),y) - CC_FLAGS_FTRACE := -fpatchable-function-entry=8 -else CC_FLAGS_FTRACE := -fpatchable-function-entry=4 +else + CC_FLAGS_FTRACE := -fpatchable-function-entry=2 endif endif diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index 04dad3380041..9e73922e1e2e 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -42,6 +42,14 @@ struct dyn_arch_ftrace { * 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * + *: + * 0: auipc t0/ra, 0x? + * 4: jalr t0/ra, ?(t0/ra) + * + *: + * 0: nop + * 4: nop + * * Dynamic ftrace generates probes to call sites, so we must deal with * both auipc and jalr at the same time. */ @@ -52,25 +60,43 @@ struct dyn_arch_ftrace { #define AUIPC_OFFSET_MASK (0xfffff000) #define AUIPC_PAD (0x00001000) #define JALR_SHIFT 20 -#define JALR_BASIC (0x000080e7) -#define AUIPC_BASIC (0x00000097) +#define JALR_RA (0x000080e7) +#define AUIPC_RA (0x00000097) +#define JALR_T0 (0x000282e7) +#define AUIPC_T0 (0x00000297) #define NOP4 (0x00000013) -#define make_call(caller, callee, call) \ +#define to_jalr_t0(offset) \ + (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0) + +#define to_auipc_t0(offset) \ + ((offset & JALR_SIGN_MASK) ? \ + (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_T0) : \ + ((offset & AUIPC_OFFSET_MASK) | AUIPC_T0)) + +#define make_call_t0(caller, callee, call) \ do { \ - call[0] = to_auipc_insn((unsigned int)((unsigned long)callee - \ - (unsigned long)caller)); \ - call[1] = to_jalr_insn((unsigned int)((unsigned long)callee - \ - (unsigned long)caller)); \ + unsigned int offset = \ + (unsigned long) callee - (unsigned long) caller; \ + call[0] = to_auipc_t0(offset); \ + call[1] = to_jalr_t0(offset); \ } while (0) -#define to_jalr_insn(offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_BASIC) +#define to_jalr_ra(offset) \ + (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) -#define to_auipc_insn(offset) \ +#define to_auipc_ra(offset) \ ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_BASIC) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_BASIC)) + (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ + ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) + +#define make_call_ra(caller, callee, call) \ +do { \ + unsigned int offset = \ + (unsigned long) callee - (unsigned long) caller; \ + call[0] = to_auipc_ra(offset); \ + call[1] = to_jalr_ra(offset); \ +} while (0) /* * Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here. diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 2086f6585773..5bff37af4770 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -55,12 +55,15 @@ static int ftrace_check_current_call(unsigned long hook_pos, } static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, - bool enable) + bool enable, bool ra) { unsigned int call[2]; unsigned int nops[2] = {NOP4, NOP4}; - make_call(hook_pos, target, call); + if (ra) + make_call_ra(hook_pos, target, call); + else + make_call_t0(hook_pos, target, call); /* Replace the auipc-jalr pair at once. Return -EPERM on write error. */ if (patch_text_nosync @@ -70,42 +73,13 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, return 0; } -/* - * Put 5 instructions with 16 bytes at the front of function within - * patchable function entry nops' area. - * - * 0: REG_S ra, -SZREG(sp) - * 1: auipc ra, 0x? - * 2: jalr -?(ra) - * 3: REG_L ra, -SZREG(sp) - * - * So the opcodes is: - * 0: 0xfe113c23 (sd)/0xfe112e23 (sw) - * 1: 0x???????? -> auipc - * 2: 0x???????? -> jalr - * 3: 0xff813083 (ld)/0xffc12083 (lw) - */ -#if __riscv_xlen == 64 -#define INSN0 0xfe113c23 -#define INSN3 0xff813083 -#elif __riscv_xlen == 32 -#define INSN0 0xfe112e23 -#define INSN3 0xffc12083 -#endif - -#define FUNC_ENTRY_SIZE 16 -#define FUNC_ENTRY_JMP 4 - int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { - unsigned int call[4] = {INSN0, 0, 0, INSN3}; - unsigned long target = addr; - unsigned long caller = rec->ip + FUNC_ENTRY_JMP; + unsigned int call[2]; - call[1] = to_auipc_insn((unsigned int)(target - caller)); - call[2] = to_jalr_insn((unsigned int)(target - caller)); + make_call_t0(rec->ip, addr, call); - if (patch_text_nosync((void *)rec->ip, call, FUNC_ENTRY_SIZE)) + if (patch_text_nosync((void *)rec->ip, call, MCOUNT_INSN_SIZE)) return -EPERM; return 0; @@ -114,15 +88,14 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { - unsigned int nops[4] = {NOP4, NOP4, NOP4, NOP4}; + unsigned int nops[2] = {NOP4, NOP4}; - if (patch_text_nosync((void *)rec->ip, nops, FUNC_ENTRY_SIZE)) + if (patch_text_nosync((void *)rec->ip, nops, MCOUNT_INSN_SIZE)) return -EPERM; return 0; } - /* * This is called early on, and isn't wrapped by * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold @@ -144,10 +117,10 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) int ftrace_update_ftrace_func(ftrace_func_t func) { int ret = __ftrace_modify_call((unsigned long)&ftrace_call, - (unsigned long)func, true); + (unsigned long)func, true, true); if (!ret) { ret = __ftrace_modify_call((unsigned long)&ftrace_regs_call, - (unsigned long)func, true); + (unsigned long)func, true, true); } return ret; @@ -159,16 +132,16 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { unsigned int call[2]; - unsigned long caller = rec->ip + FUNC_ENTRY_JMP; + unsigned long caller = rec->ip; int ret; - make_call(caller, old_addr, call); + make_call_t0(caller, old_addr, call); ret = ftrace_check_current_call(caller, call); if (ret) return ret; - return __ftrace_modify_call(caller, addr, true); + return __ftrace_modify_call(caller, addr, true, false); } #endif @@ -203,12 +176,12 @@ int ftrace_enable_ftrace_graph_caller(void) int ret; ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call, - (unsigned long)&prepare_ftrace_return, true); + (unsigned long)&prepare_ftrace_return, true, true); if (ret) return ret; return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call, - (unsigned long)&prepare_ftrace_return, true); + (unsigned long)&prepare_ftrace_return, true, true); } int ftrace_disable_ftrace_graph_caller(void) @@ -216,12 +189,12 @@ int ftrace_disable_ftrace_graph_caller(void) int ret; ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call, - (unsigned long)&prepare_ftrace_return, false); + (unsigned long)&prepare_ftrace_return, false, true); if (ret) return ret; return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call, - (unsigned long)&prepare_ftrace_return, false); + (unsigned long)&prepare_ftrace_return, false, true); } #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index d171eca623b6..125de818d1ba 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -13,8 +13,8 @@ .text -#define FENTRY_RA_OFFSET 12 -#define ABI_SIZE_ON_STACK 72 +#define FENTRY_RA_OFFSET 8 +#define ABI_SIZE_ON_STACK 80 #define ABI_A0 0 #define ABI_A1 8 #define ABI_A2 16 @@ -23,10 +23,10 @@ #define ABI_A5 40 #define ABI_A6 48 #define ABI_A7 56 -#define ABI_RA 64 +#define ABI_T0 64 +#define ABI_RA 72 .macro SAVE_ABI - addi sp, sp, -SZREG addi sp, sp, -ABI_SIZE_ON_STACK REG_S a0, ABI_A0(sp) @@ -37,6 +37,7 @@ REG_S a5, ABI_A5(sp) REG_S a6, ABI_A6(sp) REG_S a7, ABI_A7(sp) + REG_S t0, ABI_T0(sp) REG_S ra, ABI_RA(sp) .endm @@ -49,24 +50,18 @@ REG_L a5, ABI_A5(sp) REG_L a6, ABI_A6(sp) REG_L a7, ABI_A7(sp) + REG_L t0, ABI_T0(sp) REG_L ra, ABI_RA(sp) addi sp, sp, ABI_SIZE_ON_STACK - addi sp, sp, SZREG .endm #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS .macro SAVE_ALL - addi sp, sp, -SZREG addi sp, sp, -PT_SIZE_ON_STACK - REG_S x1, PT_EPC(sp) - addi sp, sp, PT_SIZE_ON_STACK - REG_L x1, (sp) - addi sp, sp, -PT_SIZE_ON_STACK + REG_S t0, PT_EPC(sp) REG_S x1, PT_RA(sp) - REG_L x1, PT_EPC(sp) - REG_S x2, PT_SP(sp) REG_S x3, PT_GP(sp) REG_S x4, PT_TP(sp) @@ -100,15 +95,11 @@ .endm .macro RESTORE_ALL + REG_L t0, PT_EPC(sp) REG_L x1, PT_RA(sp) - addi sp, sp, PT_SIZE_ON_STACK - REG_S x1, (sp) - addi sp, sp, -PT_SIZE_ON_STACK - REG_L x1, PT_EPC(sp) REG_L x2, PT_SP(sp) REG_L x3, PT_GP(sp) REG_L x4, PT_TP(sp) - REG_L x5, PT_T0(sp) REG_L x6, PT_T1(sp) REG_L x7, PT_T2(sp) REG_L x8, PT_S0(sp) @@ -137,17 +128,16 @@ REG_L x31, PT_T6(sp) addi sp, sp, PT_SIZE_ON_STACK - addi sp, sp, SZREG .endm #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ ENTRY(ftrace_caller) SAVE_ABI - addi a0, ra, -FENTRY_RA_OFFSET + addi a0, t0, -FENTRY_RA_OFFSET la a1, function_trace_op REG_L a2, 0(a1) - REG_L a1, ABI_SIZE_ON_STACK(sp) + mv a1, ra mv a3, sp ftrace_call: @@ -155,8 +145,8 @@ ftrace_call: call ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER - addi a0, sp, ABI_SIZE_ON_STACK - REG_L a1, ABI_RA(sp) + addi a0, sp, ABI_RA + REG_L a1, ABI_T0(sp) addi a1, a1, -FENTRY_RA_OFFSET #ifdef HAVE_FUNCTION_GRAPH_FP_TEST mv a2, s0 @@ -166,17 +156,17 @@ ftrace_graph_call: call ftrace_stub #endif RESTORE_ABI - ret + jr t0 ENDPROC(ftrace_caller) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS ENTRY(ftrace_regs_caller) SAVE_ALL - addi a0, ra, -FENTRY_RA_OFFSET + addi a0, t0, -FENTRY_RA_OFFSET la a1, function_trace_op REG_L a2, 0(a1) - REG_L a1, PT_SIZE_ON_STACK(sp) + mv a1, ra mv a3, sp ftrace_regs_call: @@ -196,6 +186,6 @@ ftrace_graph_regs_call: #endif RESTORE_ALL - ret + jr t0 ENDPROC(ftrace_regs_caller) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ From 8658db0a4a0f84f50fe6643fe8665efe1a4bdb66 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Fri, 13 Jan 2023 22:19:55 +0100 Subject: [PATCH 48/67] RISC-V: move some stray __RISCV_INSN_FUNCS definitions from kprobes The __RISCV_INSN_FUNCS originally declared riscv_insn_is_* functions inside the kprobes implementation. This got moved into a central header in commit ec5f90877516 ("RISC-V: Move riscv_insn_is_* macros into a common header"). Though it looks like I overlooked two of them, so fix that. FENCE itself is an instruction defined directly by its own opcode, while the created riscv_isn_is_system function covers all instructions defined under the SYSTEM opcode. Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Signed-off-by: Heiko Stuebner Link: https://lore.kernel.org/r/20230113211955.3534431-1-heiko@sntech.de Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/insn.h | 10 ++++++++++ arch/riscv/kernel/probes/simulate-insn.h | 3 --- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 25ef9c0b19e7..6567cd5ed6ba 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -128,6 +128,7 @@ #define RVC_C2_RD_OPOFF 7 /* parts of opcode for RVG*/ +#define RVG_OPCODE_FENCE 0x0f #define RVG_OPCODE_AUIPC 0x17 #define RVG_OPCODE_BRANCH 0x63 #define RVG_OPCODE_JALR 0x67 @@ -163,6 +164,7 @@ #define RVG_MATCH_AUIPC (RVG_OPCODE_AUIPC) #define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR) #define RVG_MATCH_JAL (RVG_OPCODE_JAL) +#define RVG_MATCH_FENCE (RVG_OPCODE_FENCE) #define RVG_MATCH_BEQ (RV_ENCODE_FUNCT3(BEQ) | RVG_OPCODE_BRANCH) #define RVG_MATCH_BNE (RV_ENCODE_FUNCT3(BNE) | RVG_OPCODE_BRANCH) #define RVG_MATCH_BLT (RV_ENCODE_FUNCT3(BLT) | RVG_OPCODE_BRANCH) @@ -182,6 +184,7 @@ #define RVG_MASK_AUIPC (RV_INSN_OPCODE_MASK) #define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) #define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) +#define RVG_MASK_FENCE (RV_INSN_OPCODE_MASK) #define RVC_MASK_C_JALR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) #define RVC_MASK_C_JR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) #define RVC_MASK_C_JAL (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) @@ -233,6 +236,13 @@ __RISCV_INSN_FUNCS(c_bnez, RVC_MASK_C_BNEZ, RVC_MATCH_C_BNEZ) __RISCV_INSN_FUNCS(c_ebreak, RVC_MASK_C_EBREAK, RVC_MATCH_C_EBREAK) __RISCV_INSN_FUNCS(ebreak, RVG_MASK_EBREAK, RVG_MATCH_EBREAK) __RISCV_INSN_FUNCS(sret, RVG_MASK_SRET, RVG_MATCH_SRET) +__RISCV_INSN_FUNCS(fence, RVG_MASK_FENCE, RVG_MATCH_FENCE); + +/* special case to catch _any_ system instruction */ +static __always_inline bool riscv_insn_is_system(u32 code) +{ + return (code & RV_INSN_OPCODE_MASK) == RVG_OPCODE_SYSTEM; +} /* special case to catch _any_ branch instruction */ static __always_inline bool riscv_insn_is_branch(u32 code) diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h index a19aaa0feb44..61e35db31001 100644 --- a/arch/riscv/kernel/probes/simulate-insn.h +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -12,9 +12,6 @@ } \ } while (0) -__RISCV_INSN_FUNCS(system, 0x7f, 0x73); -__RISCV_INSN_FUNCS(fence, 0x7f, 0x0f); - #define RISCV_INSN_SET_SIMULATE(name, code) \ do { \ if (riscv_insn_is_##name(code)) { \ From 7e92586c28a50dd5875655c0d169184a5acb0eee Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 19 Jan 2023 23:54:13 +0800 Subject: [PATCH 49/67] riscv: lds: define RUNTIME_DISCARD_EXIT riscv discards .exit.* sections at run-time but doesn't define RUNTIME_DISCARD_EXIT. However, the .exit.* sections are still allocated and kept even if the generic DISCARDS would discard the sections due to missing RUNTIME_DISCARD_EXIT, because the DISCARD sits at the end of the linker script. Add the missing RUNTIME_DISCARD_EXIT define so that it still works if we move DISCARD up or even at the beginning of the linker script. Signed-off-by: Jisheng Zhang Suggested-by: Masahiro Yamada Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230119155417.2600-2-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vmlinux.lds.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 4e6c88aa4d87..07c19f2a288c 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -5,6 +5,7 @@ */ #define RO_EXCEPTION_TABLE_ALIGN 4 +#define RUNTIME_DISCARD_EXIT #ifdef CONFIG_XIP_KERNEL #include "vmlinux-xip.lds.S" From e5973191a8e33ffadef4882a261130866a1e70dc Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 19 Jan 2023 23:54:14 +0800 Subject: [PATCH 50/67] riscv: vmlinux.lds.S: explicitly catch .rela.dyn symbols When enabling linker orphan section warning, I got warnings similar as below: riscv64-linux-gnu-ld: warning: orphan section `.rela.text' from `init/main.o' being placed in section `.rela.dyn' Use the approach similar as ARM64 does and declare it in vmlinux.lds.S Signed-off-by: Jisheng Zhang Link: https://lore.kernel.org/r/20230119155417.2600-3-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vmlinux.lds.S | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 07c19f2a288c..6a250313b752 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -97,6 +97,10 @@ SECTIONS *(.rel.dyn*) } + .rela.dyn : { + *(.rela*) + } + __init_data_end = .; . = ALIGN(8); From b13e64d94149c0e2981ef5d0bb496b73c48588ca Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 19 Jan 2023 23:54:15 +0800 Subject: [PATCH 51/67] riscv: vmlinux.lds.S: explicitly catch .riscv.attributes sections When enabling linker orphan section warning, I got warnings similar as below: riscv64-linux-gnu-ld: warning: orphan section `.riscv.attributes' from `init/main.o' being placed in section `.riscv.attributes' While I don't see any usage of .riscv.attributes sections' in kernel now, just catch the sections so that we can enable linker orphan section warning. Signed-off-by: Jisheng Zhang Link: https://lore.kernel.org/r/20230119155417.2600-4-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vmlinux.lds.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 6a250313b752..7eb3d25d0dae 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -146,6 +146,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG ELF_DETAILS + .riscv.attributes 0 : { *(.riscv.attributes) } DISCARDS } From 0ed0031b093b19ee46afc0a5517184932401e074 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 19 Jan 2023 23:54:16 +0800 Subject: [PATCH 52/67] riscv: vmlinux.lds.S: explicitly catch .init.bss sections from EFI stub When enabling linker orphan section warning, I got warnings similar as below: ld.lld: warning: ./drivers/firmware/efi/libstub/lib.a(efi-stub-helper.stub.o):(.init.bss) is being placed in '.init.bss' Catch the sections so that we can enable linker orphan section warning. Signed-off-by: Jisheng Zhang Link: https://lore.kernel.org/r/20230119155417.2600-5-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vmlinux.lds.S | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 7eb3d25d0dae..b2813b5e47a0 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -87,6 +87,9 @@ SECTIONS /* Start of init data section */ __init_data_begin = .; INIT_DATA_SECTION(16) + .init.bss : { + *(.init.bss) /* from the EFI stub */ + } .exit.data : { EXIT_DATA From f4b71bff8d8580eaaf56ec6c5305542a3d3a04bd Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 19 Jan 2023 23:54:17 +0800 Subject: [PATCH 53/67] riscv: select ARCH_WANT_LD_ORPHAN_WARN for !XIP_KERNEL Now, after that all the sections are explicitly described and declared in vmlinux.lds.S, we can enable ld orphan warnings for !XIP_KERNEL to prevent from missing any new sections in future. Signed-off-by: Jisheng Zhang Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230119155417.2600-6-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index e2b656043abf..335e0c45cced 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -46,6 +46,7 @@ config RISCV select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE if 64BIT + select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU select BUILDTIME_TABLE_SORT if MMU From eb165bfa8eafc3d306e34b238e53aafb35dce102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= Date: Thu, 19 Jan 2023 08:47:37 +0100 Subject: [PATCH 54/67] riscv: Add instruction dump to RISC-V splats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add instruction dump (Code:) output to RISC-V splats. Dump 16b parcels. An example: Unable to handle kernel NULL pointer dereference at virtual address 0000000000000000 Oops [#1] Modules linked in: CPU: 0 PID: 1 Comm: swapper/0 Not tainted 6.2.0-rc3-00302-g840ff44c571d-dirty #27 Hardware name: riscv-virtio,qemu (DT) epc : kernel_init+0xc8/0x10e ra : kernel_init+0x70/0x10e epc : ffffffff80bd9a40 ra : ffffffff80bd99e8 sp : ff2000000060bec0 gp : ffffffff81730b28 tp : ff6000007ff00000 t0 : 7974697275636573 t1 : 0000000000000000 t2 : 3030303270393d6e s0 : ff2000000060bee0 s1 : ffffffff81732028 a0 : 0000000000000000 a1 : ff60000080dd1780 a2 : 0000000000000002 a3 : ffffffff8176a470 a4 : 0000000000000000 a5 : 000000000000000a a6 : 0000000000000081 a7 : ff60000080dd1780 s2 : 0000000000000000 s3 : 0000000000000000 s4 : 0000000000000000 s5 : 0000000000000000 s6 : 0000000000000000 s7 : 0000000000000000 s8 : 0000000000000000 s9 : 0000000000000000 s10: 0000000000000000 s11: 0000000000000000 t3 : ffffffff81186018 t4 : 0000000000000022 t5 : 000000000000003d t6 : 0000000000000000 status: 0000000200000120 badaddr: 0000000000000000 cause: 000000000000000f [] ret_from_exception+0x0/0x16 Code: 862a d179 608c a517 0069 0513 2be5 d0ef db2e 47a9 (c11c) a517 ---[ end trace 0000000000000000 ]--- Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b SMP: stopping secondary CPUs ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b ]--- Signed-off-by: Björn Töpel Link: https://lore.kernel.org/r/20230119074738.708301-2-bjorn@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/traps.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 549bde5c970a..fbdcdbc792d3 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -29,6 +29,27 @@ int show_unhandled_signals = 1; static DEFINE_SPINLOCK(die_lock); +static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs) +{ + char str[sizeof("0000 ") * 12 + 2 + 1], *p = str; + const u16 *insns = (u16 *)instruction_pointer(regs); + long bad; + u16 val; + int i; + + for (i = -10; i < 2; i++) { + bad = get_kernel_nofault(val, &insns[i]); + if (!bad) { + p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val); + } else { + printk("%sCode: Unable to access instruction at 0x%px.\n", + loglvl, &insns[i]); + return; + } + } + printk("%sCode: %s\n", loglvl, str); +} + void die(struct pt_regs *regs, const char *str) { static int die_counter; @@ -43,8 +64,10 @@ void die(struct pt_regs *regs, const char *str) pr_emerg("%s [#%d]\n", str, ++die_counter); print_modules(); - if (regs) + if (regs) { show_regs(regs); + dump_kernel_instr(KERN_EMERG, regs); + } cause = regs ? regs->cause : -1; ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV); From 00b242509c8f2b9041c4c16d67d51eae3c9ab1d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= Date: Thu, 19 Jan 2023 08:47:38 +0100 Subject: [PATCH 55/67] scripts/decodecode: Add support for RISC-V MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RISC-V has some GNU disassembly quirks, e.g. it requires '-D' to properly disassemble .2byte directives similar to Arm [1]. Further, GNU objdump groups RISC-V instruction by 2 or 4 byte chunks, instead doing byte-for-byte. Add the required switches, and translate from short/word to bytes when ARCH is "riscv". An example how to invoke decodecode for RISC-V: $ echo 'Code: bf45 f793 1007 f7d9 50ef 37af d541 b7d9 7097 00c8 (80e7) 6140' | AFLAGS="-march=rv64imac_zicbom_zihintpause" \ ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- ./scripts/decodecode Code: bf45 f793 1007 f7d9 50ef 37af d541 b7d9 7097 00c8 (80e7) 6140 All code ======== 0: bf45 c.j 0xffffffffffffffb0 2: 1007f793 andi a5,a5,256 6: f7d9 c.bnez a5,0xffffffffffffff94 8: 37af50ef jal ra,0xf5382 c: d541 c.beqz a0,0xffffffffffffff94 e: b7d9 c.j 0xffffffffffffffd4 10: 00c87097 auipc ra,0xc87 14:* 614080e7 jalr ra,1556(ra) # 0xc87624 <-- trapping instruction Code starting with the faulting instruction =========================================== 0: 614080e7 jalr ra,1556(ra) [1] https://sourceware.org/bugzilla/show_bug.cgi?id=10263 Signed-off-by: Björn Töpel Tested-by: Alexandre Ghiti Link: https://lore.kernel.org/r/20230119074738.708301-3-bjorn@kernel.org Signed-off-by: Palmer Dabbelt --- scripts/decodecode | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/decodecode b/scripts/decodecode index b28fd2686561..8fe71c292381 100755 --- a/scripts/decodecode +++ b/scripts/decodecode @@ -93,6 +93,11 @@ disas() { ${CROSS_COMPILE}strip $t.o fi + if [ "$ARCH" = "riscv" ]; then + OBJDUMPFLAGS="-M no-aliases --section=.text -D" + ${CROSS_COMPILE}strip $t.o + fi + if [ $pc_sub -ne 0 ]; then if [ $PC ]; then adj_vma=$(( $PC - $pc_sub )) @@ -126,8 +131,13 @@ get_substr_opcode_bytes_num() do substr+="$opc" + opcode="$substr" + if [ "$ARCH" = "riscv" ]; then + opcode=$(echo $opcode | tr ' ' '\n' | tac | tr -d '\n') + fi + # return if opcode bytes do not match @opline anymore - if ! echo $opline | grep -q "$substr"; + if ! echo $opline | grep -q "$opcode"; then break fi From b49f700668fff7565b945dce823def79bff59bb0 Mon Sep 17 00:00:00 2001 From: Sergey Matyukevich Date: Mon, 30 Jan 2023 00:18:18 +0300 Subject: [PATCH 56/67] riscv: mm: fix regression due to update_mmu_cache change This is a partial revert of the commit 4bd1d80efb5a ("riscv: mm: notify remote harts about mmu cache updates"). Original commit included two loosely related changes serving the same purpose of fixing stale TLB entries causing user-space application crash: - introduce deferred per-ASID TLB flush for CPUs not running the task - switch to per-ASID TLB flush on all CPUs running the task in update_mmu_cache According to report and discussion in [1], the second part caused a regression on Renesas RZ/Five SoC. For now restore the old behavior of the update_mmu_cache. [1] https://lore.kernel.org/linux-riscv/20220829205219.283543-1-geomatsi@gmail.com/ Fixes: 4bd1d80efb5a ("riscv: mm: notify remote harts about mmu cache updates") Reported-by: "Lad, Prabhakar" Signed-off-by: Sergey Matyukevich Link: trailer, so that it can be parsed with git's trailer functionality? Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230129211818.686557-1-geomatsi@gmail.com Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 2a88362dffa5..039dee5fb76e 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -415,7 +415,7 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, * Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ - flush_tlb_page(vma, address); + local_flush_tlb_page(address); } #define __HAVE_ARCH_UPDATE_MMU_TLB From 2350bd192fa2d9ffa7a0dc7db12699c0f71af7f8 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Thu, 2 Feb 2023 15:38:32 -0800 Subject: [PATCH 57/67] RISC-V: Only provide the single-letter extensions in HWCAP The recent refactoring led to us leaking some HWCAP bits to userspace that didn't make much sense. With any luck we'll have a better scheme soon, but for now just mask off those bits to avoid polluting userspace. Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20230202233832.11036-1-palmer@rivosinc.com Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/elf.h | 10 ++++++---- arch/riscv/include/asm/hwcap.h | 8 -------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h index e7acffdf21d2..30e7d2455960 100644 --- a/arch/riscv/include/asm/elf.h +++ b/arch/riscv/include/asm/elf.h @@ -14,6 +14,7 @@ #include #include #include +#include /* * These are used to set parameters in the core dumps. @@ -59,12 +60,13 @@ extern bool compat_elf_check_arch(Elf32_Ehdr *hdr); #define STACK_RND_MASK (0x3ffff >> (PAGE_SHIFT - 12)) #endif #endif + /* - * This yields a mask that user programs can use to figure out what - * instruction set this CPU supports. This could be done in user space, - * but it's not easy, and we've already done it here. + * Provides information on the availiable set of ISA extensions to userspace, + * via a bitmap that coorespends to each single-letter ISA extension. This is + * essentially defunct, but will remain for compatibility with userspace. */ -#define ELF_HWCAP (elf_hwcap) +#define ELF_HWCAP (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)) extern unsigned long elf_hwcap; /* diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index ee9c80fe0062..831bebacb7fb 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -52,18 +52,10 @@ #include -/* - * This yields a mask that user programs can use to figure out what - * instruction set this cpu supports. - */ -#define ELF_HWCAP (elf_hwcap) - enum { CAP_HWCAP = 1, }; -extern unsigned long elf_hwcap; - struct riscv_isa_ext_data { /* Name of the extension displayed to userspace via /proc/cpuinfo */ char uprop[RISCV_ISA_EXT_NAME_LEN_MAX]; From 9ddfc3cd806081ce1f6c9c2f988cbb031f35d28f Mon Sep 17 00:00:00 2001 From: Andy Chiu Date: Mon, 6 Feb 2023 04:04:40 -0500 Subject: [PATCH 58/67] riscv: jump_label: Fixup unaligned arch_static_branch function Runtime code patching must be done at a naturally aligned address, or we may execute on a partial instruction. We have encountered problems traced back to static jump functions during the test. We switched the tracer randomly for every 1~5 seconds on a dual-core QEMU setup and found the kernel sucking at a static branch where it jumps to itself. The reason is that the static branch was 2-byte but not 4-byte aligned. Then, the kernel would patch the instruction, either J or NOP, with two half-word stores if the machine does not have efficient unaligned accesses. Thus, moments exist where half of the NOP mixes with the other half of the J when transitioning the branch. In our particular case, on a little-endian machine, the upper half of the NOP was mixed with the lower part of the J when enabling the branch, resulting in a jump that jumped to itself. Conversely, it would result in a HINT instruction when disabling the branch, but it might not be observable. ARM64 does not have this problem since all instructions must be 4-byte aligned. Fixes: ebc00dde8a97 ("riscv: Add jump-label implementation") Link: https://lore.kernel.org/linux-riscv/20220913094252.3555240-6-andy.chiu@sifive.com/ Reviewed-by: Greentime Hu Signed-off-by: Andy Chiu Signed-off-by: Guo Ren Link: https://lore.kernel.org/r/20230206090440.1255001-1-guoren@kernel.org Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/jump_label.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 6d58bbb5da46..14a5ea8d8ef0 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -18,6 +18,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) { asm_volatile_goto( + " .align 2 \n\t" " .option push \n\t" " .option norelax \n\t" " .option norvc \n\t" @@ -39,6 +40,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke const bool branch) { asm_volatile_goto( + " .align 2 \n\t" " .option push \n\t" " .option norelax \n\t" " .option norvc \n\t" From 1eac28201ac0725192f5ced34192d281a06692e5 Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Wed, 8 Feb 2023 23:53:27 +0100 Subject: [PATCH 59/67] RISC-V: fix ordering of Zbb extension As Andrew reported, Zb* comes after Zi* according 27.11 "Subset Naming Convention" so fix the ordering accordingly. Reported-by: Andrew Jones Signed-off-by: Heiko Stuebner Reviewed-by: Conor Dooley Reviewed-by: Andrew Jones Tested-by: Conor Dooley Link: https://lore.kernel.org/r/20230208225328.1636017-2-heiko@sntech.de Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 420228e219f7..8400f0cc9704 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -185,9 +185,9 @@ arch_initcall(riscv_cpuinfo_init); * New entries to this struct should follow the ordering rules described above. */ static struct riscv_isa_ext_data isa_ext_arr[] = { - __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), + __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), From dac8bf14bb49aecd1de99ebb5498fa03152f2d40 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Thu, 9 Feb 2023 13:36:36 +0100 Subject: [PATCH 60/67] riscv: hwcap: Don't alphabetize ISA extension IDs While the comment above the ISA extension ID definitions says "Entries are sorted alphabetically.", this stopped being good advice with commit d8a3d8a75206 ("riscv: hwcap: make ISA extension ids can be used in asm"), as we now use macros instead of enums. Reshuffling defines is error-prone, so, since they don't need to be in any particular order, change the advice to just adding new extensions at the bottom. Also, take the opportunity to change spaces to tabs, merge three comments into one, and move the base and max defines into more logical locations wrt the ID definitions. Signed-off-by: Andrew Jones Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230209123636.123537-1-ajones@ventanamicro.com Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/hwcap.h | 38 ++++++++++++++++------------------ 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 831bebacb7fb..8f3994a7f0ca 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -24,29 +24,27 @@ #define RISCV_ISA_EXT_u ('u' - 'a') /* - * Increse this to higher value as kernel support more ISA extensions. + * These macros represent the logical IDs of each multi-letter RISC-V ISA + * extension and are used in the ISA bitmap. The logical IDs start from + * RISCV_ISA_EXT_BASE, which allows the 0-25 range to be reserved for single + * letter extensions. The maximum, RISCV_ISA_EXT_MAX, is defined in order + * to allocate the bitmap and may be increased when necessary. + * + * New extensions should just be added to the bottom, rather than added + * alphabetically, in order to avoid unnecessary shuffling. */ -#define RISCV_ISA_EXT_MAX 64 -#define RISCV_ISA_EXT_NAME_LEN_MAX 32 +#define RISCV_ISA_EXT_BASE 26 -/* The base ID for multi-letter ISA extensions */ -#define RISCV_ISA_EXT_BASE 26 +#define RISCV_ISA_EXT_SSCOFPMF 26 +#define RISCV_ISA_EXT_SSTC 27 +#define RISCV_ISA_EXT_SVINVAL 28 +#define RISCV_ISA_EXT_SVPBMT 29 +#define RISCV_ISA_EXT_ZBB 30 +#define RISCV_ISA_EXT_ZICBOM 31 +#define RISCV_ISA_EXT_ZIHINTPAUSE 32 -/* - * These macros represent the logical ID for each multi-letter RISC-V ISA extension. - * The logical ID should start from RISCV_ISA_EXT_BASE and must not exceed - * RISCV_ISA_EXT_MAX. 0-25 range is reserved for single letter - * extensions while all the multi-letter extensions should define the next - * available logical extension id. - * Entries are sorted alphabetically. - */ -#define RISCV_ISA_EXT_SSCOFPMF 26 -#define RISCV_ISA_EXT_SSTC 27 -#define RISCV_ISA_EXT_SVINVAL 28 -#define RISCV_ISA_EXT_SVPBMT 29 -#define RISCV_ISA_EXT_ZBB 30 -#define RISCV_ISA_EXT_ZICBOM 31 -#define RISCV_ISA_EXT_ZIHINTPAUSE 32 +#define RISCV_ISA_EXT_MAX 64 +#define RISCV_ISA_EXT_NAME_LEN_MAX 32 #ifndef __ASSEMBLY__ From 9493e6f3ce02f44c21aa19f3cbf3b9aa05479d06 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Sun, 12 Feb 2023 19:47:36 +0000 Subject: [PATCH 61/67] RISC-V: take text_mutex during alternative patching Guenter reported a splat during boot, that Samuel pointed out was the lockdep assertion failing in patch_insn_write(): WARNING: CPU: 0 PID: 0 at arch/riscv/kernel/patch.c:63 patch_insn_write+0x222/0x2f6 epc : patch_insn_write+0x222/0x2f6 ra : patch_insn_write+0x21e/0x2f6 epc : ffffffff800068c6 ra : ffffffff800068c2 sp : ffffffff81803df0 gp : ffffffff81a1ab78 tp : ffffffff81814f80 t0 : ffffffffffffe000 t1 : 0000000000000001 t2 : 4c45203a76637369 s0 : ffffffff81803e40 s1 : 0000000000000004 a0 : 0000000000000000 a1 : ffffffffffffffff a2 : 0000000000000004 a3 : 0000000000000000 a4 : 0000000000000001 a5 : 0000000000000000 a6 : 0000000000000000 a7 : 0000000052464e43 s2 : ffffffff80b4889c s3 : 000000000000082c s4 : ffffffff80b48828 s5 : 0000000000000828 s6 : ffffffff8131a0a0 s7 : 0000000000000fff s8 : 0000000008000200 s9 : ffffffff8131a520 s10: 0000000000000018 s11: 000000000000000b t3 : 0000000000000001 t4 : 000000000000000d t5 : ffffffffd8180000 t6 : ffffffff81803bc8 status: 0000000200000100 badaddr: 0000000000000000 cause: 0000000000000003 [] patch_insn_write+0x222/0x2f6 [] patch_text_nosync+0xc/0x2a [] riscv_cpufeature_patch_func+0x52/0x98 [] _apply_alternatives+0x46/0x86 [] apply_boot_alternatives+0x3c/0xfa [] setup_arch+0x584/0x5b8 [] start_kernel+0xa2/0x8f8 This issue was exposed by 702e64550b12 ("riscv: fpu: switch has_fpu() to riscv_has_extension_likely()"), as it is the patching in has_fpu() that triggers the splats in Guenter's report. Take the text_mutex before doing any code patching to satisfy lockdep. Fixes: ff689fd21cb1 ("riscv: add RISC-V Svpbmt extension support") Fixes: a35707c3d850 ("riscv: add memory-type errata for T-Head") Fixes: 1a0e5dbd3723 ("riscv: sifive: Add SiFive alternative ports") Reported-by: Guenter Roeck Link: https://lore.kernel.org/all/20230212154333.GA3760469@roeck-us.net/ Signed-off-by: Conor Dooley Reviewed-by: Samuel Holland Tested-by: Guenter Roeck Link: https://lore.kernel.org/r/20230212194735.491785-1-conor@kernel.org Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/errata/sifive/errata.c | 3 +++ arch/riscv/errata/thead/errata.c | 8 ++++++-- arch/riscv/kernel/cpufeature.c | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index ef9a4eec0dba..da55cb247e89 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -107,8 +108,10 @@ void __init_or_module sifive_errata_patch_func(struct alt_entry *begin, tmp = (1U << alt->errata_id); if (cpu_req_errata & tmp) { + mutex_lock(&text_mutex); patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt), alt->alt_len); + mutex_lock(&text_mutex); cpu_apply_errata |= tmp; } } diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c index 1dd90a5f86f0..3b96a06d3c54 100644 --- a/arch/riscv/errata/thead/errata.c +++ b/arch/riscv/errata/thead/errata.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -101,10 +102,13 @@ void __init_or_module thead_errata_patch_func(struct alt_entry *begin, struct al altptr = ALT_ALT_PTR(alt); /* On vm-alternatives, the mmu isn't running yet */ - if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) { memcpy(oldptr, altptr, alt->alt_len); - else + } else { + mutex_lock(&text_mutex); patch_text_nosync(oldptr, altptr, alt->alt_len); + mutex_unlock(&text_mutex); + } } } diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 21fb567e1b22..59d58ee0f68d 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -292,8 +293,11 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, oldptr = ALT_OLD_PTR(alt); altptr = ALT_ALT_PTR(alt); + + mutex_lock(&text_mutex); patch_text_nosync(oldptr, altptr, alt->alt_len); riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); + mutex_unlock(&text_mutex); } } #endif From 416721ff05fddc58ca531b6f069de250301de6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= Date: Tue, 14 Feb 2023 17:25:15 +0100 Subject: [PATCH 62/67] riscv, mm: Perform BPF exhandler fixup on page fault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 21855cac82d3 ("riscv/mm: Prevent kernel module to access user memory without uaccess routines") added early exits/deaths for page faults stemming from accesses to user-space without using proper uaccess routines (where sstatus.SUM is set). Unfortunatly, this is too strict for some BPF programs, which relies on BPF exhandler fixups. These BPF programs loads "BTF pointers". A BTF pointers could either be a valid kernel pointer or NULL, but not a userspace address. Resolve the problem by calling the fixup handler in the early exit path. Fixes: 21855cac82d3 ("riscv/mm: Prevent kernel module to access user memory without uaccess routines") Signed-off-by: Björn Töpel Link: https://lore.kernel.org/r/20230214162515.184827-1-bjorn@kernel.org Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/fault.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index d86f7cebd4a7..eb0774d9c03b 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -267,10 +267,12 @@ asmlinkage void do_page_fault(struct pt_regs *regs) if (user_mode(regs)) flags |= FAULT_FLAG_USER; - if (!user_mode(regs) && addr < TASK_SIZE && - unlikely(!(regs->status & SR_SUM))) - die_kernel_fault("access to user memory without uaccess routines", - addr, regs); + if (!user_mode(regs) && addr < TASK_SIZE && unlikely(!(regs->status & SR_SUM))) { + if (fixup_exception(regs)) + return; + + die_kernel_fault("access to user memory without uaccess routines", addr, regs); + } perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); From 130aee3fd9981297ff9354e5d5609cd59aafbbea Mon Sep 17 00:00:00 2001 From: Mattias Nissler Date: Wed, 15 Feb 2023 14:48:28 +0000 Subject: [PATCH 63/67] riscv: Avoid enabling interrupts in die() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While working on something else, I noticed that the kernel would start accepting interrupts again after crashing in an interrupt handler. Since the kernel is already in inconsistent state, enabling interrupts is dangerous and opens up risk of kernel state deteriorating further. Interrupts do get enabled via what looks like an unintended side effect of spin_unlock_irq, so switch to the more cautious spin_lock_irqsave/spin_unlock_irqrestore instead. Fixes: 76d2a0493a17 ("RISC-V: Init and Halt Code") Signed-off-by: Mattias Nissler Reviewed-by: Björn Töpel Link: https://lore.kernel.org/r/20230215144828.3370316-1-mnissler@rivosinc.com Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/traps.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 549bde5c970a..70c98ce23be2 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -34,10 +34,11 @@ void die(struct pt_regs *regs, const char *str) static int die_counter; int ret; long cause; + unsigned long flags; oops_enter(); - spin_lock_irq(&die_lock); + spin_lock_irqsave(&die_lock, flags); console_verbose(); bust_spinlocks(1); @@ -54,7 +55,7 @@ void die(struct pt_regs *regs, const char *str) bust_spinlocks(0); add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); - spin_unlock_irq(&die_lock); + spin_unlock_irqrestore(&die_lock, flags); oops_exit(); if (in_interrupt()) From 91612cfb173619b38087449cd3811f9cb4420d94 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Mon, 16 Jan 2023 00:28:11 +0800 Subject: [PATCH 64/67] riscv: alternative: proceed one more instruction for auipc/jalr pair If we patched auipc + jalr pair, we'd better proceed one more instruction. Andrew pointed out "There's not a problem now, since we're only adding a fixup for jal, not jalr, but we should future-proof this and there's no reason to revisit an already fixed-up instruction anyway." Signed-off-by: Jisheng Zhang Suggested-by: Andrew Jones Reviewed-by: Andrew Jones Reviewed-by: Heiko Stuebner Link: https://lore.kernel.org/r/20230115162811.3146-1-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/alternative.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index fc341b69bf62..2354c69dc7d1 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -122,6 +122,7 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), insn, insn2, patch_offset); + i++; } if (riscv_insn_is_jal(insn)) { From 8ac6e619d9d51b3eb5bae817db8aa94e780a0db4 Mon Sep 17 00:00:00 2001 From: Liao Chang Date: Sun, 29 Jan 2023 17:42:42 +0800 Subject: [PATCH 65/67] riscv: Add header include guards to insn.h Add header include guards to insn.h to prevent repeating declaration of any identifiers in insn.h. Fixes: edde5584c7ab ("riscv: Add SW single-step support for KDB") Signed-off-by: Liao Chang Reviewed-by: Andrew Jones Fixes: c9c1af3f186a ("RISC-V: rename parse_asm.h to insn.h") Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20230129094242.282620-1-liaochang1@huawei.com Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/insn.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 6567cd5ed6ba..8d5c84f2d5ef 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -3,6 +3,9 @@ * Copyright (C) 2020 SiFive */ +#ifndef _ASM_RISCV_INSN_H +#define _ASM_RISCV_INSN_H + #include #define RV_INSN_FUNCT3_MASK GENMASK(14, 12) @@ -375,3 +378,4 @@ static inline void riscv_insn_insert_utype_itype_imm(u32 *utype_insn, u32 *itype *utype_insn |= (imm & RV_U_IMM_31_12_MASK) + ((imm & BIT(11)) << 1); *itype_insn |= ((imm & RV_I_IMM_11_0_MASK) << RV_I_IMM_11_0_OPOFF); } +#endif /* _ASM_RISCV_INSN_H */ From a3c7d6b642e4b4619f220bf43e8b78c34ffaecf1 Mon Sep 17 00:00:00 2001 From: Guo Ren Date: Tue, 31 Jan 2023 20:52:59 -0500 Subject: [PATCH 66/67] riscv: mm: hugetlb: Enable ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP Add HVO support for RISC-V; see commit 6be24bed9da3 ("mm: hugetlb: introduce a new config HUGETLB_PAGE_FREE_VMEMMAP"). This patch is similar to commit 1e63ac088f20 ("arm64: mm: hugetlb: enable HUGETLB_PAGE_FREE_VMEMMAP for arm64"), and riscv's motivation is the same as arm64. The current riscv was ready to enable HVO after fixup, ref commit d33deda095d3 ("riscv/mm: hugepage's PG_dcache_clean flag is only set in head page"). See Documentation/mm/vmemmap_dedup.rst for more details. The HugeTLB VmemmapvOptimization (HVO) defaults to off in Kconfig. Here is the riscv test log: cat /proc/sys/vm/hugetlb_optimize_vmemmap echo 8 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages mount -t hugetlbfs none test/ -o pagesize=2048k Signed-off-by: Guo Ren Signed-off-by: Guo Ren Link: https://lore.kernel.org/linux-riscv/1F5AF29D-708A-483B-A29F-CAEE6F554866@linux.dev/ Acked-by: Muchun Song Reviewed-by: Andrew Jones Link: https://lore.kernel.org/r/20230201015259.3222524-1-guoren@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 6292baab1516..81eb031887d2 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -46,6 +46,7 @@ config RISCV select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_GENERAL_HUGETLB + select ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE From eb9be8310c58c166f9fae3b71c0ad9d6741b4897 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Fri, 10 Feb 2023 18:59:45 +0000 Subject: [PATCH 67/67] RISC-V: add a spin_shadow_stack declaration The patchwork automation reported a sparse complaint that spin_shadow_stack was not declared and should be static: ../arch/riscv/kernel/traps.c:335:15: warning: symbol 'spin_shadow_stack' was not declared. Should it be static? However, this is used in entry.S and therefore shouldn't be static. The same applies to the shadow_stack that this pseudo spinlock is trying to protect, so do like its charge and add a declaration to thread_info.h Signed-off-by: Conor Dooley Fixes: 7e1864332fbc ("riscv: fix race when vmap stack overflow") Reviewed-by: Guo Ren Link: https://lore.kernel.org/r/20230210185945.915806-1-conor@kernel.org Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/thread_info.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 67322f878e0d..f704c8dd57e0 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -43,6 +43,7 @@ #ifndef __ASSEMBLY__ extern long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE / sizeof(long)]; +extern unsigned long spin_shadow_stack; #include #include