From e5d1c8722083f0332dcd3c85fa1273d85fb6bed8 Mon Sep 17 00:00:00 2001 From: Nikita Zhandarovich Date: Tue, 18 Apr 2023 06:07:43 -0700 Subject: [PATCH 01/42] PM: domains: fix integer overflow issues in genpd_parse_state() Currently, while calculating residency and latency values, right operands may overflow if resulting values are big enough. To prevent this, albeit unlikely case, play it safe and convert right operands to left ones' type s64. Found by Linux Verification Center (linuxtesting.org) with static analysis tool SVACE. Fixes: 30f604283e05 ("PM / Domains: Allow domain power states to be read from DT") Signed-off-by: Nikita Zhandarovich Acked-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 32084e38b73d..51b9d4eaab5e 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2939,10 +2939,10 @@ static int genpd_parse_state(struct genpd_power_state *genpd_state, err = of_property_read_u32(state_node, "min-residency-us", &residency); if (!err) - genpd_state->residency_ns = 1000 * residency; + genpd_state->residency_ns = 1000LL * residency; - genpd_state->power_on_latency_ns = 1000 * exit_latency; - genpd_state->power_off_latency_ns = 1000 * entry_latency; + genpd_state->power_on_latency_ns = 1000LL * exit_latency; + genpd_state->power_off_latency_ns = 1000LL * entry_latency; genpd_state->fwnode = &state_node->fwnode; return 0; From 31cb1304ad8bd27b7d2abd8669fb887fb47d8eaf Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:05 +0800 Subject: [PATCH 02/42] powercap: intel_rapl: Remove unused field in struct rapl_if_priv After commit f1e8d7560d30 ("powercap/intel_rapl: enumerate Psys RAPL domain together with package RAPL domain"), the platform_rapl_domain field is not used anymore. Remove it from rapl_if_priv structure. Fixes: f1e8d7560d30 ("powercap/intel_rapl: enumerate Psys RAPL domain together with package RAPL domain") Signed-off-by: Zhang Rui Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 9f4b6f5b822f..828557645770 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -124,7 +124,6 @@ struct reg_action { */ struct rapl_if_priv { struct powercap_control_type *control_type; - struct rapl_domain *platform_rapl_domain; enum cpuhp_state pcap_rapl_online; u64 reg_unit; u64 regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; From 1488ac990ac886b1209aa9f94c0c66022bcc8827 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:06 +0800 Subject: [PATCH 03/42] powercap: intel_rapl: Allow probing without CPUID match Currently, CPU model checks is used to 1. get proper rapl_defaults callbacks for RAPL MSR/MMIO Interface. 2. create a platform device node for the intel_rapl_msr driver to probe. Both of these are only mandatory for the RAPL MSR/MMIO Interface. Make the CPUID match optional. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 35 +++++++++++----------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 8970c7b80884..2d2c15eea8eb 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1528,32 +1528,25 @@ static int __init rapl_init(void) int ret; id = x86_match_cpu(rapl_ids); - if (!id) { - pr_err("driver does not support CPU family %d model %d\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); + if (id) { + rapl_defaults = (struct rapl_defaults *)id->driver_data; - return -ENODEV; + rapl_msr_platdev = platform_device_alloc("intel_rapl_msr", 0); + if (!rapl_msr_platdev) + return -ENOMEM; + + ret = platform_device_add(rapl_msr_platdev); + if (ret) { + platform_device_put(rapl_msr_platdev); + return ret; + } } - rapl_defaults = (struct rapl_defaults *)id->driver_data; - ret = register_pm_notifier(&rapl_pm_notifier); - if (ret) - return ret; - - rapl_msr_platdev = platform_device_alloc("intel_rapl_msr", 0); - if (!rapl_msr_platdev) { - ret = -ENOMEM; - goto end; - } - - ret = platform_device_add(rapl_msr_platdev); - if (ret) + if (ret && rapl_msr_platdev) { + platform_device_del(rapl_msr_platdev); platform_device_put(rapl_msr_platdev); - -end: - if (ret) - unregister_pm_notifier(&rapl_pm_notifier); + } return ret; } From e8e28c2af16b279b6c37d533e1e73effb197cf2e Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:07 +0800 Subject: [PATCH 04/42] powercap: intel_rapl: Support per Interface rapl_defaults rapl_defaults is Interface specific. Although current MSR and MMIO Interface share the same rapl_defaults, new Interface like TPMI need its own rapl_defaults callbacks. Save the rapl_defaults information in the Interface private structure. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 46 ++++++++++++++++++++-------- include/linux/intel_rapl.h | 2 ++ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 2d2c15eea8eb..59e470a57342 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -115,6 +115,11 @@ struct rapl_defaults { }; static struct rapl_defaults *rapl_defaults; +static struct rapl_defaults *get_defaults(struct rapl_package *rp) +{ + return rp->priv->defaults; +} + /* Sideband MBI registers */ #define IOSF_CPU_POWER_BUDGET_CTL_BYT (0x2) #define IOSF_CPU_POWER_BUDGET_CTL_TNG (0xdf) @@ -227,14 +232,15 @@ static int find_nr_power_limit(struct rapl_domain *rd) static int set_domain_enable(struct powercap_zone *power_zone, bool mode) { struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); + struct rapl_defaults *defaults = get_defaults(rd->rp); if (rd->state & DOMAIN_STATE_BIOS_LOCKED) return -EACCES; cpus_read_lock(); rapl_write_data_raw(rd, PL1_ENABLE, mode); - if (rapl_defaults->set_floor_freq) - rapl_defaults->set_floor_freq(rd, mode); + if (defaults->set_floor_freq) + defaults->set_floor_freq(rd, mode); cpus_read_unlock(); return 0; @@ -551,6 +557,7 @@ static void rapl_init_domains(struct rapl_package *rp) enum rapl_domain_type i; enum rapl_domain_reg_id j; struct rapl_domain *rd = rp->domains; + struct rapl_defaults *defaults = get_defaults(rp); for (i = 0; i < RAPL_DOMAIN_MAX; i++) { unsigned int mask = rp->domain_map & (1 << i); @@ -592,14 +599,14 @@ static void rapl_init_domains(struct rapl_package *rp) switch (i) { case RAPL_DOMAIN_DRAM: rd->domain_energy_unit = - rapl_defaults->dram_domain_energy_unit; + defaults->dram_domain_energy_unit; if (rd->domain_energy_unit) pr_info("DRAM domain energy unit %dpj\n", rd->domain_energy_unit); break; case RAPL_DOMAIN_PLATFORM: rd->domain_energy_unit = - rapl_defaults->psys_domain_energy_unit; + defaults->psys_domain_energy_unit; if (rd->domain_energy_unit) pr_info("Platform domain energy unit %dpj\n", rd->domain_energy_unit); @@ -616,6 +623,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, { u64 units = 1; struct rapl_package *rp = rd->rp; + struct rapl_defaults *defaults = get_defaults(rp); u64 scale = 1; switch (type) { @@ -631,7 +639,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, units = rp->energy_unit; break; case TIME_UNIT: - return rapl_defaults->compute_time_window(rp, value, to_raw); + return defaults->compute_time_window(rp, value, to_raw); case ARBITRARY_UNIT: default: return value; @@ -702,10 +710,18 @@ static struct rapl_primitive_info rpi[] = { {NULL, 0, 0, 0}, }; +static int rapl_config(struct rapl_package *rp) +{ + rp->priv->defaults = (void *)rapl_defaults; + return 0; +} + static enum rapl_primitives prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim) { - if (!rapl_defaults->spr_psys_bits) + struct rapl_defaults *defaults = get_defaults(rd->rp); + + if (!defaults->spr_psys_bits) return prim; if (rd->id != RAPL_DOMAIN_PLATFORM) @@ -960,16 +976,17 @@ static void set_floor_freq_default(struct rapl_domain *rd, bool mode) static void set_floor_freq_atom(struct rapl_domain *rd, bool enable) { static u32 power_ctrl_orig_val; + struct rapl_defaults *defaults = get_defaults(rd->rp); u32 mdata; - if (!rapl_defaults->floor_freq_reg_addr) { + if (!defaults->floor_freq_reg_addr) { pr_err("Invalid floor frequency config register\n"); return; } if (!power_ctrl_orig_val) iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ, - rapl_defaults->floor_freq_reg_addr, + defaults->floor_freq_reg_addr, &power_ctrl_orig_val); mdata = power_ctrl_orig_val; if (enable) { @@ -977,7 +994,7 @@ static void set_floor_freq_atom(struct rapl_domain *rd, bool enable) mdata |= 1 << 8; } iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE, - rapl_defaults->floor_freq_reg_addr, mdata); + defaults->floor_freq_reg_addr, mdata); } static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, @@ -1385,11 +1402,9 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) { int id = topology_logical_die_id(cpu); struct rapl_package *rp; + struct rapl_defaults *defaults; int ret; - if (!rapl_defaults) - return ERR_PTR(-ENODEV); - rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL); if (!rp) return ERR_PTR(-ENOMEM); @@ -1399,6 +1414,10 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) rp->lead_cpu = cpu; rp->priv = priv; + ret = rapl_config(rp); + if (ret) + goto err_free_package; + if (topology_max_die_per_package() > 1) snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d", @@ -1407,8 +1426,9 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", topology_physical_package_id(cpu)); + defaults = get_defaults(rp); /* check if the package contains valid domains */ - if (rapl_detect_domains(rp, cpu) || rapl_defaults->check_unit(rp, cpu)) { + if (rapl_detect_domains(rp, cpu) || defaults->check_unit(rp, cpu)) { ret = -ENODEV; goto err_free_package; } diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 828557645770..ebd1cad78212 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -121,6 +121,7 @@ struct reg_action { * registers. * @write_raw: Callback for writing RAPL interface specific * registers. + * @defaults: internal pointer to interface default settings */ struct rapl_if_priv { struct powercap_control_type *control_type; @@ -130,6 +131,7 @@ struct rapl_if_priv { int limits[RAPL_DOMAIN_MAX]; int (*read_raw)(int cpu, struct reg_action *ra); int (*write_raw)(int cpu, struct reg_action *ra); + void *defaults; }; /* maximum rapl package domain name: package-%d-die-%d */ From 98ff639a7289067247b3ef9dd5d1e922361e7365 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:08 +0800 Subject: [PATCH 05/42] powercap: intel_rapl: Support per Interface primitive information RAPL primitive information is Interface specific. Although current MSR and MMIO Interface share the same RAPL primitives, new Interface like TPMI has its own RAPL primitive information. Save the primitive information in the Interface private structure. Plus, using variant name "rp" for struct rapl_primitive_info is confusing because "rp" is also used for struct rapl_package. Use "rpi" as the variant name for struct rapl_primitive_info, and rename the previous rpi[] array to avoid conflict. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 50 ++++++++++++++++++---------- include/linux/intel_rapl.h | 2 ++ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 59e470a57342..267ec36b1649 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -654,7 +654,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, } /* in the order of enum rapl_primitives */ -static struct rapl_primitive_info rpi[] = { +static struct rapl_primitive_info rpi_default[] = { /* name, mask, shift, msr index, unit divisor */ PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0, RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0), @@ -710,9 +710,20 @@ static struct rapl_primitive_info rpi[] = { {NULL, 0, 0, 0}, }; +static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim) +{ + struct rapl_primitive_info *rpi = rp->priv->rpi; + + if (prim < 0 || prim > NR_RAPL_PRIMITIVES || !rpi) + return NULL; + + return &rpi[prim]; +} + static int rapl_config(struct rapl_package *rp) { rp->priv->defaults = (void *)rapl_defaults; + rp->priv->rpi = (void *)rpi_default; return 0; } @@ -763,14 +774,14 @@ static int rapl_read_data_raw(struct rapl_domain *rd, { u64 value; enum rapl_primitives prim_fixed = prim_fixups(rd, prim); - struct rapl_primitive_info *rp = &rpi[prim_fixed]; + struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); struct reg_action ra; int cpu; - if (!rp->name || rp->flag & RAPL_PRIMITIVE_DUMMY) + if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) return -EINVAL; - ra.reg = rd->regs[rp->id]; + ra.reg = rd->regs[rpi->id]; if (!ra.reg) return -EINVAL; @@ -778,26 +789,26 @@ static int rapl_read_data_raw(struct rapl_domain *rd, /* domain with 2 limits has different bit */ if (prim == FW_LOCK && rd->rp->priv->limits[rd->id] == 2) { - rp->mask = POWER_HIGH_LOCK; - rp->shift = 63; + rpi->mask = POWER_HIGH_LOCK; + rpi->shift = 63; } /* non-hardware data are collected by the polling thread */ - if (rp->flag & RAPL_PRIMITIVE_DERIVED) { + if (rpi->flag & RAPL_PRIMITIVE_DERIVED) { *data = rd->rdd.primitives[prim]; return 0; } - ra.mask = rp->mask; + ra.mask = rpi->mask; if (rd->rp->priv->read_raw(cpu, &ra)) { pr_debug("failed to read reg 0x%llx on cpu %d\n", ra.reg, cpu); return -EIO; } - value = ra.value >> rp->shift; + value = ra.value >> rpi->shift; if (xlate) - *data = rapl_unit_xlate(rd, rp->unit, value, 0); + *data = rapl_unit_xlate(rd, rpi->unit, value, 0); else *data = value; @@ -810,21 +821,24 @@ static int rapl_write_data_raw(struct rapl_domain *rd, unsigned long long value) { enum rapl_primitives prim_fixed = prim_fixups(rd, prim); - struct rapl_primitive_info *rp = &rpi[prim_fixed]; + struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); int cpu; u64 bits; struct reg_action ra; int ret; + if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) + return -EINVAL; + cpu = rd->rp->lead_cpu; - bits = rapl_unit_xlate(rd, rp->unit, value, 1); - bits <<= rp->shift; - bits &= rp->mask; + bits = rapl_unit_xlate(rd, rpi->unit, value, 1); + bits <<= rpi->shift; + bits &= rpi->mask; memset(&ra, 0, sizeof(ra)); - ra.reg = rd->regs[rp->id]; - ra.mask = rp->mask; + ra.reg = rd->regs[rpi->id]; + ra.mask = rpi->mask; ra.value = bits; ret = rd->rp->priv->write_raw(cpu, &ra); @@ -1176,8 +1190,10 @@ static void rapl_update_domain_data(struct rapl_package *rp) rp->domains[dmn].name); /* exclude non-raw primitives */ for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) { + struct rapl_primitive_info *rpi = get_rpi(rp, prim); + if (!rapl_read_data_raw(&rp->domains[dmn], prim, - rpi[prim].unit, &val)) + rpi->unit, &val)) rp->domains[dmn].rdd.primitives[prim] = val; } } diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index ebd1cad78212..f51e2df7130e 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -122,6 +122,7 @@ struct reg_action { * @write_raw: Callback for writing RAPL interface specific * registers. * @defaults: internal pointer to interface default settings + * @rpi: internal pointer to interface primitive info */ struct rapl_if_priv { struct powercap_control_type *control_type; @@ -132,6 +133,7 @@ struct rapl_if_priv { int (*read_raw)(int cpu, struct reg_action *ra); int (*write_raw)(int cpu, struct reg_action *ra); void *defaults; + void *rpi; }; /* maximum rapl package domain name: package-%d-die-%d */ From cb532e728ee2880be53264752e74945fd2d917ac Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:09 +0800 Subject: [PATCH 06/42] powercap: intel_rapl: Support per domain energy/power/time unit RAPL MSR/MMIO Interface has package scope unit register but some RAPL domains like Dram/Psys may use a fixed energy unit value instead of the default unit value on certain platforms. RAPL TPMI Interface supports per domain unit register. For the above reasons, add support for per domain unit register and per domain energy/power/time unit. When per domain unit register is not available, use the package scope unit register as the per domain unit register for each RAPL domain so that this change is transparent to MSR/MMIO Interface. No functional change intended. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 128 +++++++++++++++------------ include/linux/intel_rapl.h | 8 +- 2 files changed, 73 insertions(+), 63 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 267ec36b1649..3625d4466cb3 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -105,9 +105,9 @@ static const char pl4_name[] = "peak_power"; struct rapl_defaults { u8 floor_freq_reg_addr; - int (*check_unit)(struct rapl_package *rp, int cpu); + int (*check_unit)(struct rapl_domain *rd, int cpu); void (*set_floor_freq)(struct rapl_domain *rd, bool mode); - u64 (*compute_time_window)(struct rapl_package *rp, u64 val, + u64 (*compute_time_window)(struct rapl_domain *rd, u64 val, bool to_raw); unsigned int dram_domain_energy_unit; unsigned int psys_domain_energy_unit; @@ -557,7 +557,6 @@ static void rapl_init_domains(struct rapl_package *rp) enum rapl_domain_type i; enum rapl_domain_reg_id j; struct rapl_domain *rd = rp->domains; - struct rapl_defaults *defaults = get_defaults(rp); for (i = 0; i < RAPL_DOMAIN_MAX; i++) { unsigned int mask = rp->domain_map & (1 << i); @@ -596,24 +595,6 @@ static void rapl_init_domains(struct rapl_package *rp) for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++) rd->regs[j] = rp->priv->regs[i][j]; - switch (i) { - case RAPL_DOMAIN_DRAM: - rd->domain_energy_unit = - defaults->dram_domain_energy_unit; - if (rd->domain_energy_unit) - pr_info("DRAM domain energy unit %dpj\n", - rd->domain_energy_unit); - break; - case RAPL_DOMAIN_PLATFORM: - rd->domain_energy_unit = - defaults->psys_domain_energy_unit; - if (rd->domain_energy_unit) - pr_info("Platform domain energy unit %dpj\n", - rd->domain_energy_unit); - break; - default: - break; - } rd++; } } @@ -622,24 +603,19 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, u64 value, int to_raw) { u64 units = 1; - struct rapl_package *rp = rd->rp; - struct rapl_defaults *defaults = get_defaults(rp); + struct rapl_defaults *defaults = get_defaults(rd->rp); u64 scale = 1; switch (type) { case POWER_UNIT: - units = rp->power_unit; + units = rd->power_unit; break; case ENERGY_UNIT: scale = ENERGY_UNIT_SCALE; - /* per domain unit takes precedence */ - if (rd->domain_energy_unit) - units = rd->domain_energy_unit; - else - units = rp->energy_unit; + units = rd->energy_unit; break; case TIME_UNIT: - return defaults->compute_time_window(rp, value, to_raw); + return defaults->compute_time_window(rd, value, to_raw); case ARBITRARY_UNIT: default: return value; @@ -857,58 +833,58 @@ static int rapl_write_data_raw(struct rapl_domain *rd, * power unit : microWatts : Represented in milliWatts by default * time unit : microseconds: Represented in seconds by default */ -static int rapl_check_unit_core(struct rapl_package *rp, int cpu) +static int rapl_check_unit_core(struct rapl_domain *rd, int cpu) { struct reg_action ra; u32 value; - ra.reg = rp->priv->reg_unit; + ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rp->priv->read_raw(cpu, &ra)) { + if (rd->rp->priv->read_raw(cpu, &ra)) { pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", - rp->priv->reg_unit, cpu); + ra.reg, cpu); return -ENODEV; } value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; - rp->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); + rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; - rp->power_unit = 1000000 / (1 << value); + rd->power_unit = 1000000 / (1 << value); value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; - rp->time_unit = 1000000 / (1 << value); + rd->time_unit = 1000000 / (1 << value); - pr_debug("Core CPU %s energy=%dpJ, time=%dus, power=%duW\n", - rp->name, rp->energy_unit, rp->time_unit, rp->power_unit); + pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n", + rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); return 0; } -static int rapl_check_unit_atom(struct rapl_package *rp, int cpu) +static int rapl_check_unit_atom(struct rapl_domain *rd, int cpu) { struct reg_action ra; u32 value; - ra.reg = rp->priv->reg_unit; + ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rp->priv->read_raw(cpu, &ra)) { + if (rd->rp->priv->read_raw(cpu, &ra)) { pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", - rp->priv->reg_unit, cpu); + ra.reg, cpu); return -ENODEV; } value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; - rp->energy_unit = ENERGY_UNIT_SCALE * 1 << value; + rd->energy_unit = ENERGY_UNIT_SCALE * 1 << value; value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; - rp->power_unit = (1 << value) * 1000; + rd->power_unit = (1 << value) * 1000; value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; - rp->time_unit = 1000000 / (1 << value); + rd->time_unit = 1000000 / (1 << value); - pr_debug("Atom %s energy=%dpJ, time=%dus, power=%duW\n", - rp->name, rp->energy_unit, rp->time_unit, rp->power_unit); + pr_debug("Atom %s:%s energy=%dpJ, time=%dus, power=%duW\n", + rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); return 0; } @@ -1011,7 +987,7 @@ static void set_floor_freq_atom(struct rapl_domain *rd, bool enable) defaults->floor_freq_reg_addr, mdata); } -static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, +static u64 rapl_compute_time_window_core(struct rapl_domain *rd, u64 value, bool to_raw) { u64 f, y; /* fraction and exp. used for time unit */ @@ -1023,12 +999,12 @@ static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, if (!to_raw) { f = (value & 0x60) >> 5; y = value & 0x1f; - value = (1 << y) * (4 + f) * rp->time_unit / 4; + value = (1 << y) * (4 + f) * rd->time_unit / 4; } else { - if (value < rp->time_unit) + if (value < rd->time_unit) return 0; - do_div(value, rp->time_unit); + do_div(value, rd->time_unit); y = ilog2(value); /* @@ -1044,7 +1020,7 @@ static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, return value; } -static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value, +static u64 rapl_compute_time_window_atom(struct rapl_domain *rd, u64 value, bool to_raw) { /* @@ -1052,9 +1028,9 @@ static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value, * where time_unit is default to 1 sec. Never 0. */ if (!to_raw) - return (value) ? value * rp->time_unit : rp->time_unit; + return (value) ? value * rd->time_unit : rd->time_unit; - value = div64_u64(value, rp->time_unit); + value = div64_u64(value, rd->time_unit); return value; } @@ -1299,6 +1275,40 @@ static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp) return 0; } +/* + * Get per domain energy/power/time unit. + * RAPL Interfaces without per domain unit register will use the package + * scope unit register to set per domain units. + */ +static int rapl_get_domain_unit(struct rapl_domain *rd) +{ + struct rapl_defaults *defaults = get_defaults(rd->rp); + int ret; + + if (!rd->regs[RAPL_DOMAIN_REG_UNIT]) { + if (!rd->rp->priv->reg_unit) { + pr_err("No valid Unit register found\n"); + return -ENODEV; + } + rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit; + } + + if (!defaults->check_unit) { + pr_err("missing .check_unit() callback\n"); + return -ENODEV; + } + + ret = defaults->check_unit(rd, rd->rp->lead_cpu); + if (ret) + return ret; + + if (rd->id == RAPL_DOMAIN_DRAM && defaults->dram_domain_energy_unit) + rd->energy_unit = defaults->dram_domain_energy_unit; + if (rd->id == RAPL_DOMAIN_PLATFORM && defaults->psys_domain_energy_unit) + rd->energy_unit = defaults->psys_domain_energy_unit; + return 0; +} + /* * Check if power limits are available. Two cases when they are not available: * 1. Locked by BIOS, in this case we still provide read-only access so that @@ -1359,8 +1369,10 @@ static int rapl_detect_domains(struct rapl_package *rp, int cpu) rapl_init_domains(rp); - for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) + for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { + rapl_get_domain_unit(rd); rapl_detect_powerlimit(rd); + } return 0; } @@ -1418,7 +1430,6 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) { int id = topology_logical_die_id(cpu); struct rapl_package *rp; - struct rapl_defaults *defaults; int ret; rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL); @@ -1442,9 +1453,8 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", topology_physical_package_id(cpu)); - defaults = get_defaults(rp); /* check if the package contains valid domains */ - if (rapl_detect_domains(rp, cpu) || defaults->check_unit(rp, cpu)) { + if (rapl_detect_domains(rp, cpu)) { ret = -ENODEV; goto err_free_package; } diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index f51e2df7130e..936fb8c3082c 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -30,6 +30,7 @@ enum rapl_domain_reg_id { RAPL_DOMAIN_REG_POLICY, RAPL_DOMAIN_REG_INFO, RAPL_DOMAIN_REG_PL4, + RAPL_DOMAIN_REG_UNIT, RAPL_DOMAIN_REG_MAX, }; @@ -96,7 +97,9 @@ struct rapl_domain { struct rapl_power_limit rpl[NR_POWER_LIMITS]; u64 attr_map; /* track capabilities */ unsigned int state; - unsigned int domain_energy_unit; + unsigned int power_unit; + unsigned int energy_unit; + unsigned int time_unit; struct rapl_package *rp; }; @@ -143,9 +146,6 @@ struct rapl_package { unsigned int id; /* logical die id, equals physical 1-die systems */ unsigned int nr_domains; unsigned long domain_map; /* bit map of active domains */ - unsigned int power_unit; - unsigned int energy_unit; - unsigned int time_unit; struct rapl_domain *domains; /* array of domains, sized at runtime */ struct powercap_zone *power_zone; /* keep track of parent zone */ unsigned long power_limit_irq; /* keep track of package power limit From 11edbe5c66d624e2e1eec8929d3668d76a574c3b Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:10 +0800 Subject: [PATCH 07/42] powercap: intel_rapl: Use index to initialize primitive information Currently, the RAPL primitive information array is required to be initialized in the order of enum rapl_primitives. This can break easily, especially when different RAPL Interfaces may support different sets of primitives. Convert the code to initialize the primitive information using array index explicitly. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 54 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 3625d4466cb3..97d9e1e628e2 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -629,61 +629,59 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, return div64_u64(value, scale); } -/* in the order of enum rapl_primitives */ -static struct rapl_primitive_info rpi_default[] = { +static struct rapl_primitive_info rpi_default[NR_RAPL_PRIMITIVES] = { /* name, mask, shift, msr index, unit divisor */ - PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0, + [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0, RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0), - PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0, + [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32, + [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0, + [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0, RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31, + [FW_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15, + [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16, + [PL1_CLAMP] = PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47, + [PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48, + [PL2_CLAMP] = PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PL4_ENABLE, POWER_LIMIT4_MASK, 0, + [PL4_ENABLE] = PRIMITIVE_INFO_INIT(PL4_ENABLE, POWER_LIMIT4_MASK, 0, RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17, + [TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17, RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), - PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49, + [TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49, RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), - PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK, + [THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK, 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32, + [MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16, + [MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48, + [MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48, RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0), - PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0, + [THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0, RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0), - PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0, + [PRIORITY_LEVEL] = PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0, RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT1, PSYS_POWER_LIMIT1_MASK, 0, + [PSYS_POWER_LIMIT1] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT1, PSYS_POWER_LIMIT1_MASK, 0, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT2, PSYS_POWER_LIMIT2_MASK, 32, + [PSYS_POWER_LIMIT2] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT2, PSYS_POWER_LIMIT2_MASK, 32, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(PSYS_PL1_ENABLE, PSYS_POWER_LIMIT1_ENABLE, 17, + [PSYS_PL1_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL1_ENABLE, PSYS_POWER_LIMIT1_ENABLE, 17, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PSYS_PL2_ENABLE, PSYS_POWER_LIMIT2_ENABLE, 49, + [PSYS_PL2_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL2_ENABLE, PSYS_POWER_LIMIT2_ENABLE, 49, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), - PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW1_MASK, 19, + [PSYS_TIME_WINDOW1] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW1_MASK, 19, RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), - PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW2, PSYS_TIME_WINDOW2_MASK, 51, + [PSYS_TIME_WINDOW2] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW2, PSYS_TIME_WINDOW2_MASK, 51, RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), /* non-hardware */ - PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT, + [AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT, RAPL_PRIMITIVE_DERIVED), - {NULL, 0, 0, 0}, }; static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim) From 045610c383bd6b740bb7e7c780d6f7729249e60d Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:11 +0800 Subject: [PATCH 08/42] powercap: intel_rapl: Change primitive order The same set of operations are shared by different Powert Limits, including Power Limit get/set, Power Limit enable/disable, clamping enable/disable, time window get/set, and max power get/set, etc. But the same operation for different Power Limit has different primitives because they use different registers/register bits. A lot of dirty/duplicate code was introduced to handle this difference. Instead of using hardcoded primitive name directly, using Power Limit id + operation type is much cleaner. For this sense, move POWER_LIMIT1/POWER_LIMIT2/POWER_LIMIT4 to the beginning of enum rapl_primitives so that they can be reused as Power Limit ids. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 4 ++-- include/linux/intel_rapl.h | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 97d9e1e628e2..154f93b3dee5 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -631,14 +631,14 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, static struct rapl_primitive_info rpi_default[NR_RAPL_PRIMITIVES] = { /* name, mask, shift, msr index, unit divisor */ - [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0, - RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0), [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0, RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0), + [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0, + RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0), [FW_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15, diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 936fb8c3082c..bbd03b17dc8d 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -37,10 +37,10 @@ enum rapl_domain_reg_id { struct rapl_domain; enum rapl_primitives { - ENERGY_COUNTER, POWER_LIMIT1, POWER_LIMIT2, POWER_LIMIT4, + ENERGY_COUNTER, FW_LOCK, PL1_ENABLE, /* power limit 1, aka long term */ @@ -75,7 +75,8 @@ struct rapl_domain_data { unsigned long timestamp; }; -#define NR_POWER_LIMITS (3) +#define NR_POWER_LIMITS (POWER_LIMIT4 + 1) + struct rapl_power_limit { struct powercap_zone_constraint *constraint; int prim_id; /* primitive ID used to enable */ From a38f300bb23c896d2d132a4502086d4bfec2a25e Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:12 +0800 Subject: [PATCH 09/42] powercap: intel_rapl: Use bitmap for Power Limits Currently, a RAPL package is registered with the number of Power Limits supported in each RAPL domain. But this doesn't tell which Power Limits are available. Using the number of Power Limits supported to guess the availability of each Power Limit is fragile. Use bitmap to represent the availability of each Power Limit. Note that PL1 is mandatory thus it does not need to be set explicitly by the RAPL Interface drivers. No functional change intended. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 14 ++++++-------- drivers/powercap/intel_rapl_msr.c | 6 +++--- .../intel/int340x_thermal/processor_thermal_rapl.c | 4 ++-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 154f93b3dee5..8e77df42257a 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -574,20 +574,18 @@ static void rapl_init_domains(struct rapl_package *rp) rapl_domain_names[i]); rd->id = i; + + /* PL1 is supported by default */ + rp->priv->limits[i] |= BIT(POWER_LIMIT1); rd->rpl[0].prim_id = PL1_ENABLE; rd->rpl[0].name = pl1_name; - /* - * The PL2 power domain is applicable for limits two - * and limits three - */ - if (rp->priv->limits[i] >= 2) { + if (rp->priv->limits[i] & BIT(POWER_LIMIT2)) { rd->rpl[1].prim_id = PL2_ENABLE; rd->rpl[1].name = pl2_name; } - /* Enable PL4 domain if the total power limits are three */ - if (rp->priv->limits[i] == 3) { + if (rp->priv->limits[i] & BIT(POWER_LIMIT4)) { rd->rpl[2].prim_id = PL4_ENABLE; rd->rpl[2].name = pl4_name; } @@ -762,7 +760,7 @@ static int rapl_read_data_raw(struct rapl_domain *rd, cpu = rd->rp->lead_cpu; /* domain with 2 limits has different bit */ - if (prim == FW_LOCK && rd->rp->priv->limits[rd->id] == 2) { + if (prim == FW_LOCK && (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))) { rpi->mask = POWER_HIGH_LOCK; rpi->shift = 63; } diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index a27673706c3d..6fe5e556aa51 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -45,8 +45,8 @@ static struct rapl_if_priv rapl_msr_priv_intel = { MSR_DRAM_POWER_LIMIT, MSR_DRAM_ENERGY_STATUS, MSR_DRAM_PERF_STATUS, 0, MSR_DRAM_POWER_INFO }, .regs[RAPL_DOMAIN_PLATFORM] = { MSR_PLATFORM_POWER_LIMIT, MSR_PLATFORM_ENERGY_STATUS, 0, 0, 0}, - .limits[RAPL_DOMAIN_PACKAGE] = 2, - .limits[RAPL_DOMAIN_PLATFORM] = 2, + .limits[RAPL_DOMAIN_PACKAGE] = BIT(POWER_LIMIT2), + .limits[RAPL_DOMAIN_PLATFORM] = BIT(POWER_LIMIT2), }; static struct rapl_if_priv rapl_msr_priv_amd = { @@ -169,7 +169,7 @@ static int rapl_msr_probe(struct platform_device *pdev) rapl_msr_priv->write_raw = rapl_msr_write_raw; if (id) { - rapl_msr_priv->limits[RAPL_DOMAIN_PACKAGE] = 3; + rapl_msr_priv->limits[RAPL_DOMAIN_PACKAGE] |= BIT(POWER_LIMIT4); rapl_msr_priv->regs[RAPL_DOMAIN_PACKAGE][RAPL_DOMAIN_REG_PL4] = MSR_VR_CURRENT_CONFIG; pr_info("PL4 support detected.\n"); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c index a205221ec8df..e070239106f5 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c @@ -15,8 +15,8 @@ static const struct rapl_mmio_regs rapl_mmio_default = { .reg_unit = 0x5938, .regs[RAPL_DOMAIN_PACKAGE] = { 0x59a0, 0x593c, 0x58f0, 0, 0x5930}, .regs[RAPL_DOMAIN_DRAM] = { 0x58e0, 0x58e8, 0x58ec, 0, 0}, - .limits[RAPL_DOMAIN_PACKAGE] = 2, - .limits[RAPL_DOMAIN_DRAM] = 2, + .limits[RAPL_DOMAIN_PACKAGE] = BIT(POWER_LIMIT2), + .limits[RAPL_DOMAIN_DRAM] = BIT(POWER_LIMIT2), }; static int rapl_mmio_cpu_online(unsigned int cpu) From 9050a9cd5e4c848e265915d6e7b1f731e6e1e0e6 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:13 +0800 Subject: [PATCH 10/42] powercap: intel_rapl: Cleanup Power Limits support The same set of operations are shared by different Powert Limits, including Power Limit get/set, Power Limit enable/disable, clamping enable/disable, time window get/set, and max power get/set, etc. But the same operation for different Power Limit has different primitives because they use different registers/register bits. A lot of dirty/duplicate code was introduced to handle this difference. Introduce a universal way to issue Power Limit operations. Instead of using hardcoded primitive name directly, use Power Limit id + operation type, and hide all the Power Limit difference details in a central place, get_pl_prim(). Two helpers, rapl_read_pl_data() and rapl_write_pl_data(), are introduced at the same time to simplify the code for issuing Power Limit operations. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 343 ++++++++++++--------------- include/linux/intel_rapl.h | 1 - 2 files changed, 146 insertions(+), 198 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 8e77df42257a..7f80c35e5c86 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -96,9 +96,67 @@ enum unit_type { #define DOMAIN_STATE_POWER_LIMIT_SET BIT(1) #define DOMAIN_STATE_BIOS_LOCKED BIT(2) -static const char pl1_name[] = "long_term"; -static const char pl2_name[] = "short_term"; -static const char pl4_name[] = "peak_power"; +static const char *pl_names[NR_POWER_LIMITS] = { + [POWER_LIMIT1] = "long_term", + [POWER_LIMIT2] = "short_term", + [POWER_LIMIT4] = "peak_power", +}; + +enum pl_prims { + PL_ENABLE, + PL_CLAMP, + PL_LIMIT, + PL_TIME_WINDOW, + PL_MAX_POWER, +}; + +static bool is_pl_valid(struct rapl_domain *rd, int pl) +{ + if (pl < POWER_LIMIT1 || pl > POWER_LIMIT4) + return false; + return rd->rpl[pl].name ? true : false; +} + +static int get_pl_prim(int pl, enum pl_prims prim) +{ + switch (pl) { + case POWER_LIMIT1: + if (prim == PL_ENABLE) + return PL1_ENABLE; + if (prim == PL_CLAMP) + return PL1_CLAMP; + if (prim == PL_LIMIT) + return POWER_LIMIT1; + if (prim == PL_TIME_WINDOW) + return TIME_WINDOW1; + if (prim == PL_MAX_POWER) + return THERMAL_SPEC_POWER; + return -EINVAL; + case POWER_LIMIT2: + if (prim == PL_ENABLE) + return PL2_ENABLE; + if (prim == PL_CLAMP) + return PL2_CLAMP; + if (prim == PL_LIMIT) + return POWER_LIMIT2; + if (prim == PL_TIME_WINDOW) + return TIME_WINDOW2; + if (prim == PL_MAX_POWER) + return MAX_POWER; + return -EINVAL; + case POWER_LIMIT4: + if (prim == PL_LIMIT) + return POWER_LIMIT4; + if (prim == PL_ENABLE) + return PL4_ENABLE; + /* PL4 would be around two times PL2, use same prim as PL2. */ + if (prim == PL_MAX_POWER) + return MAX_POWER; + return -EINVAL; + default: + return -EINVAL; + } +} #define power_zone_to_rapl_domain(_zone) \ container_of(_zone, struct rapl_domain, power_zone) @@ -155,6 +213,12 @@ static int rapl_read_data_raw(struct rapl_domain *rd, static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value); +static int rapl_read_pl_data(struct rapl_domain *rd, int pl, + enum pl_prims pl_prim, + bool xlate, u64 *data); +static int rapl_write_pl_data(struct rapl_domain *rd, int pl, + enum pl_prims pl_prim, + unsigned long long value); static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, u64 value, int to_raw); static void package_power_limit_irq_save(struct rapl_package *rp); @@ -222,7 +286,7 @@ static int find_nr_power_limit(struct rapl_domain *rd) int i, nr_pl = 0; for (i = 0; i < NR_POWER_LIMITS; i++) { - if (rd->rpl[i].name) + if (is_pl_valid(rd, i)) nr_pl++; } @@ -233,37 +297,34 @@ static int set_domain_enable(struct powercap_zone *power_zone, bool mode) { struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); struct rapl_defaults *defaults = get_defaults(rd->rp); - - if (rd->state & DOMAIN_STATE_BIOS_LOCKED) - return -EACCES; + int ret; cpus_read_lock(); - rapl_write_data_raw(rd, PL1_ENABLE, mode); - if (defaults->set_floor_freq) + ret = rapl_write_pl_data(rd, POWER_LIMIT1, PL_ENABLE, mode); + if (!ret && defaults->set_floor_freq) defaults->set_floor_freq(rd, mode); cpus_read_unlock(); - return 0; + return ret; } static int get_domain_enable(struct powercap_zone *power_zone, bool *mode) { struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); u64 val; + int ret; if (rd->state & DOMAIN_STATE_BIOS_LOCKED) { *mode = false; return 0; } cpus_read_lock(); - if (rapl_read_data_raw(rd, PL1_ENABLE, true, &val)) { - cpus_read_unlock(); - return -EIO; - } - *mode = val; + ret = rapl_read_pl_data(rd, POWER_LIMIT1, PL_ENABLE, true, &val); + if (!ret) + *mode = val; cpus_read_unlock(); - return 0; + return ret; } /* per RAPL domain ops, in the order of rapl_domain_type */ @@ -319,8 +380,8 @@ static int contraint_to_pl(struct rapl_domain *rd, int cid) { int i, j; - for (i = 0, j = 0; i < NR_POWER_LIMITS; i++) { - if ((rd->rpl[i].name) && j++ == cid) { + for (i = POWER_LIMIT1, j = 0; i < NR_POWER_LIMITS; i++) { + if (is_pl_valid(rd, i) && j++ == cid) { pr_debug("%s: index %d\n", __func__, i); return i; } @@ -341,36 +402,11 @@ static int set_power_limit(struct powercap_zone *power_zone, int cid, cpus_read_lock(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); - if (id < 0) { - ret = id; - goto set_exit; - } - rp = rd->rp; - if (rd->state & DOMAIN_STATE_BIOS_LOCKED) { - dev_warn(&power_zone->dev, - "%s locked by BIOS, monitoring only\n", rd->name); - ret = -EACCES; - goto set_exit; - } - - switch (rd->rpl[id].prim_id) { - case PL1_ENABLE: - rapl_write_data_raw(rd, POWER_LIMIT1, power_limit); - break; - case PL2_ENABLE: - rapl_write_data_raw(rd, POWER_LIMIT2, power_limit); - break; - case PL4_ENABLE: - rapl_write_data_raw(rd, POWER_LIMIT4, power_limit); - break; - default: - ret = -EINVAL; - } + ret = rapl_write_pl_data(rd, id, PL_LIMIT, power_limit); if (!ret) package_power_limit_irq_save(rp); -set_exit: cpus_read_unlock(); return ret; } @@ -380,38 +416,17 @@ static int get_current_power_limit(struct powercap_zone *power_zone, int cid, { struct rapl_domain *rd; u64 val; - int prim; int ret = 0; int id; cpus_read_lock(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); - if (id < 0) { - ret = id; - goto get_exit; - } - switch (rd->rpl[id].prim_id) { - case PL1_ENABLE: - prim = POWER_LIMIT1; - break; - case PL2_ENABLE: - prim = POWER_LIMIT2; - break; - case PL4_ENABLE: - prim = POWER_LIMIT4; - break; - default: - cpus_read_unlock(); - return -EINVAL; - } - if (rapl_read_data_raw(rd, prim, true, &val)) - ret = -EIO; - else + ret = rapl_read_pl_data(rd, id, PL_LIMIT, true, &val); + if (!ret) *data = val; -get_exit: cpus_read_unlock(); return ret; @@ -427,23 +442,9 @@ static int set_time_window(struct powercap_zone *power_zone, int cid, cpus_read_lock(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); - if (id < 0) { - ret = id; - goto set_time_exit; - } - switch (rd->rpl[id].prim_id) { - case PL1_ENABLE: - rapl_write_data_raw(rd, TIME_WINDOW1, window); - break; - case PL2_ENABLE: - rapl_write_data_raw(rd, TIME_WINDOW2, window); - break; - default: - ret = -EINVAL; - } + ret = rapl_write_pl_data(rd, id, PL_TIME_WINDOW, window); -set_time_exit: cpus_read_unlock(); return ret; } @@ -459,33 +460,11 @@ static int get_time_window(struct powercap_zone *power_zone, int cid, cpus_read_lock(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); - if (id < 0) { - ret = id; - goto get_time_exit; - } - switch (rd->rpl[id].prim_id) { - case PL1_ENABLE: - ret = rapl_read_data_raw(rd, TIME_WINDOW1, true, &val); - break; - case PL2_ENABLE: - ret = rapl_read_data_raw(rd, TIME_WINDOW2, true, &val); - break; - case PL4_ENABLE: - /* - * Time window parameter is not applicable for PL4 entry - * so assigining '0' as default value. - */ - val = 0; - break; - default: - cpus_read_unlock(); - return -EINVAL; - } + ret = rapl_read_pl_data(rd, id, PL_TIME_WINDOW, true, &val); if (!ret) *data = val; -get_time_exit: cpus_read_unlock(); return ret; @@ -505,36 +484,23 @@ static const char *get_constraint_name(struct powercap_zone *power_zone, return NULL; } -static int get_max_power(struct powercap_zone *power_zone, int id, u64 *data) +static int get_max_power(struct powercap_zone *power_zone, int cid, u64 *data) { struct rapl_domain *rd; u64 val; - int prim; int ret = 0; + int id; cpus_read_lock(); rd = power_zone_to_rapl_domain(power_zone); - switch (rd->rpl[id].prim_id) { - case PL1_ENABLE: - prim = THERMAL_SPEC_POWER; - break; - case PL2_ENABLE: - prim = MAX_POWER; - break; - case PL4_ENABLE: - prim = MAX_POWER; - break; - default: - cpus_read_unlock(); - return -EINVAL; - } - if (rapl_read_data_raw(rd, prim, true, &val)) - ret = -EIO; - else + id = contraint_to_pl(rd, cid); + + ret = rapl_read_pl_data(rd, id, PL_MAX_POWER, true, &val); + if (!ret) *data = val; /* As a generalization rule, PL4 would be around two times PL2. */ - if (rd->rpl[id].prim_id == PL4_ENABLE) + if (id == POWER_LIMIT4) *data = *data * 2; cpus_read_unlock(); @@ -560,6 +526,7 @@ static void rapl_init_domains(struct rapl_package *rp) for (i = 0; i < RAPL_DOMAIN_MAX; i++) { unsigned int mask = rp->domain_map & (1 << i); + int t; if (!mask) continue; @@ -577,17 +544,10 @@ static void rapl_init_domains(struct rapl_package *rp) /* PL1 is supported by default */ rp->priv->limits[i] |= BIT(POWER_LIMIT1); - rd->rpl[0].prim_id = PL1_ENABLE; - rd->rpl[0].name = pl1_name; - if (rp->priv->limits[i] & BIT(POWER_LIMIT2)) { - rd->rpl[1].prim_id = PL2_ENABLE; - rd->rpl[1].name = pl2_name; - } - - if (rp->priv->limits[i] & BIT(POWER_LIMIT4)) { - rd->rpl[2].prim_id = PL4_ENABLE; - rd->rpl[2].name = pl4_name; + for (t = POWER_LIMIT1; t < NR_POWER_LIMITS; t++) { + if (rp->priv->limits[i] & BIT(t)) + rd->rpl[t].name = pl_names[t]; } for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++) @@ -818,6 +778,33 @@ static int rapl_write_data_raw(struct rapl_domain *rd, return ret; } +static int rapl_read_pl_data(struct rapl_domain *rd, int pl, + enum pl_prims pl_prim, bool xlate, u64 *data) +{ + enum rapl_primitives prim = get_pl_prim(pl, pl_prim); + + if (!is_pl_valid(rd, pl)) + return -EINVAL; + + return rapl_read_data_raw(rd, prim, xlate, data); +} + +static int rapl_write_pl_data(struct rapl_domain *rd, int pl, + enum pl_prims pl_prim, + unsigned long long value) +{ + enum rapl_primitives prim = get_pl_prim(pl, pl_prim); + + if (!is_pl_valid(rd, pl)) + return -EINVAL; + + if (rd->state & DOMAIN_STATE_BIOS_LOCKED) { + pr_warn("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]); + return -EACCES; + } + + return rapl_write_data_raw(rd, prim, value); +} /* * Raw RAPL data stored in MSRs are in certain scales. We need to * convert them into standard units based on the units reported in @@ -945,17 +932,16 @@ static void package_power_limit_irq_restore(struct rapl_package *rp) static void set_floor_freq_default(struct rapl_domain *rd, bool mode) { - int nr_powerlimit = find_nr_power_limit(rd); + int i; /* always enable clamp such that p-state can go below OS requested * range. power capping priority over guranteed frequency. */ - rapl_write_data_raw(rd, PL1_CLAMP, mode); + rapl_write_pl_data(rd, POWER_LIMIT1, PL_CLAMP, mode); - /* some domains have pl2 */ - if (nr_powerlimit > 1) { - rapl_write_data_raw(rd, PL2_ENABLE, mode); - rapl_write_data_raw(rd, PL2_CLAMP, mode); + for (i = POWER_LIMIT2; i < NR_POWER_LIMITS; i++) { + rapl_write_pl_data(rd, i, PL_ENABLE, mode); + rapl_write_pl_data(rd, i, PL_CLAMP, mode); } } @@ -1327,11 +1313,10 @@ static void rapl_detect_powerlimit(struct rapl_domain *rd) rd->state |= DOMAIN_STATE_BIOS_LOCKED; } } - /* check if power limit MSR exists, otherwise domain is monitoring only */ - for (i = 0; i < NR_POWER_LIMITS; i++) { - int prim = rd->rpl[i].prim_id; - if (rapl_read_data_raw(rd, prim, false, &val64)) + /* check if power limit exists, otherwise domain is monitoring only */ + for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) { + if (rapl_read_pl_data(rd, i, PL_ENABLE, false, &val64)) rd->rpl[i].name = NULL; } } @@ -1381,13 +1366,13 @@ void rapl_remove_package(struct rapl_package *rp) package_power_limit_irq_restore(rp); for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { - rapl_write_data_raw(rd, PL1_ENABLE, 0); - rapl_write_data_raw(rd, PL1_CLAMP, 0); - if (find_nr_power_limit(rd) > 1) { - rapl_write_data_raw(rd, PL2_ENABLE, 0); - rapl_write_data_raw(rd, PL2_CLAMP, 0); - rapl_write_data_raw(rd, PL4_ENABLE, 0); + int i; + + for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) { + rapl_write_pl_data(rd, i, PL_ENABLE, 0); + rapl_write_pl_data(rd, i, PL_CLAMP, 0); } + if (rd->id == RAPL_DOMAIN_PACKAGE) { rd_package = rd; continue; @@ -1472,38 +1457,18 @@ static void power_limit_state_save(void) { struct rapl_package *rp; struct rapl_domain *rd; - int nr_pl, ret, i; + int ret, i; cpus_read_lock(); list_for_each_entry(rp, &rapl_packages, plist) { if (!rp->power_zone) continue; rd = power_zone_to_rapl_domain(rp->power_zone); - nr_pl = find_nr_power_limit(rd); - for (i = 0; i < nr_pl; i++) { - switch (rd->rpl[i].prim_id) { - case PL1_ENABLE: - ret = rapl_read_data_raw(rd, - POWER_LIMIT1, true, + for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) { + ret = rapl_read_pl_data(rd, i, PL_LIMIT, true, &rd->rpl[i].last_power_limit); - if (ret) - rd->rpl[i].last_power_limit = 0; - break; - case PL2_ENABLE: - ret = rapl_read_data_raw(rd, - POWER_LIMIT2, true, - &rd->rpl[i].last_power_limit); - if (ret) - rd->rpl[i].last_power_limit = 0; - break; - case PL4_ENABLE: - ret = rapl_read_data_raw(rd, - POWER_LIMIT4, true, - &rd->rpl[i].last_power_limit); - if (ret) - rd->rpl[i].last_power_limit = 0; - break; - } + if (ret) + rd->rpl[i].last_power_limit = 0; } } cpus_read_unlock(); @@ -1513,33 +1478,17 @@ static void power_limit_state_restore(void) { struct rapl_package *rp; struct rapl_domain *rd; - int nr_pl, i; + int i; cpus_read_lock(); list_for_each_entry(rp, &rapl_packages, plist) { if (!rp->power_zone) continue; rd = power_zone_to_rapl_domain(rp->power_zone); - nr_pl = find_nr_power_limit(rd); - for (i = 0; i < nr_pl; i++) { - switch (rd->rpl[i].prim_id) { - case PL1_ENABLE: - if (rd->rpl[i].last_power_limit) - rapl_write_data_raw(rd, POWER_LIMIT1, - rd->rpl[i].last_power_limit); - break; - case PL2_ENABLE: - if (rd->rpl[i].last_power_limit) - rapl_write_data_raw(rd, POWER_LIMIT2, - rd->rpl[i].last_power_limit); - break; - case PL4_ENABLE: - if (rd->rpl[i].last_power_limit) - rapl_write_data_raw(rd, POWER_LIMIT4, - rd->rpl[i].last_power_limit); - break; - } - } + for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) + if (rd->rpl[i].last_power_limit) + rapl_write_pl_data(rd, i, PL_LIMIT, + rd->rpl[i].last_power_limit); } cpus_read_unlock(); } diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index bbd03b17dc8d..df17f4e51dbf 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -79,7 +79,6 @@ struct rapl_domain_data { struct rapl_power_limit { struct powercap_zone_constraint *constraint; - int prim_id; /* primitive ID used to enable */ struct rapl_domain *domain; const char *name; u64 last_power_limit; From f442bd2742174eed6993315ec621275df13f311d Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:14 +0800 Subject: [PATCH 11/42] powercap: intel_rapl: Add support for lock bit per Power Limit With RAPL MSR/MMIO Interface, each RAPL domain has one Power Limit register. Each Power Limit register has one lock bit which tells the OS if the power limit register can be used or not. Depending on the number of power limits supported by the power limit register, the lock bit may apply to one or more power limits. With RAPL TPMI Interface, each RAPL domain has multiple Power Limits, and each Power Limit has its own register, with a lock bit. To handle this, introduce support for lock bit per Power Limit. For existing RAPL MSR/MMIO Interfaces, the lock bit in the Power Limit register applies to all the Power Limits controlled by this register. Remove the per domain DOMAIN_STATE_BIOS_LOCKED flag at the same time because it can be replaced by the per Power Limit lock. No functional change intended. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 52 +++++++++++++++++----------- include/linux/intel_rapl.h | 2 ++ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 7f80c35e5c86..d88008308d7a 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -94,7 +94,6 @@ enum unit_type { #define DOMAIN_STATE_INACTIVE BIT(0) #define DOMAIN_STATE_POWER_LIMIT_SET BIT(1) -#define DOMAIN_STATE_BIOS_LOCKED BIT(2) static const char *pl_names[NR_POWER_LIMITS] = { [POWER_LIMIT1] = "long_term", @@ -108,6 +107,7 @@ enum pl_prims { PL_LIMIT, PL_TIME_WINDOW, PL_MAX_POWER, + PL_LOCK, }; static bool is_pl_valid(struct rapl_domain *rd, int pl) @@ -117,7 +117,18 @@ static bool is_pl_valid(struct rapl_domain *rd, int pl) return rd->rpl[pl].name ? true : false; } -static int get_pl_prim(int pl, enum pl_prims prim) +static int get_pl_lock_prim(struct rapl_domain *rd, int pl) +{ + /* + * Power Limit register that supports two power limits has a different + * bit position for the Lock bit. + */ + if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2)) + return FW_HIGH_LOCK; + return FW_LOCK; +} + +static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim) { switch (pl) { case POWER_LIMIT1: @@ -131,6 +142,8 @@ static int get_pl_prim(int pl, enum pl_prims prim) return TIME_WINDOW1; if (prim == PL_MAX_POWER) return THERMAL_SPEC_POWER; + if (prim == PL_LOCK) + return get_pl_lock_prim(rd, pl); return -EINVAL; case POWER_LIMIT2: if (prim == PL_ENABLE) @@ -143,6 +156,8 @@ static int get_pl_prim(int pl, enum pl_prims prim) return TIME_WINDOW2; if (prim == PL_MAX_POWER) return MAX_POWER; + if (prim == PL_LOCK) + return get_pl_lock_prim(rd, pl); return -EINVAL; case POWER_LIMIT4: if (prim == PL_LIMIT) @@ -314,7 +329,7 @@ static int get_domain_enable(struct powercap_zone *power_zone, bool *mode) u64 val; int ret; - if (rd->state & DOMAIN_STATE_BIOS_LOCKED) { + if (rd->rpl[POWER_LIMIT1].locked) { *mode = false; return 0; } @@ -599,6 +614,8 @@ static struct rapl_primitive_info rpi_default[NR_RAPL_PRIMITIVES] = { RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0), [FW_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), + [FW_HIGH_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_HIGH_LOCK, 63, + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), [PL1_CLAMP] = PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16, @@ -719,11 +736,6 @@ static int rapl_read_data_raw(struct rapl_domain *rd, cpu = rd->rp->lead_cpu; - /* domain with 2 limits has different bit */ - if (prim == FW_LOCK && (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))) { - rpi->mask = POWER_HIGH_LOCK; - rpi->shift = 63; - } /* non-hardware data are collected by the polling thread */ if (rpi->flag & RAPL_PRIMITIVE_DERIVED) { *data = rd->rdd.primitives[prim]; @@ -781,7 +793,7 @@ static int rapl_write_data_raw(struct rapl_domain *rd, static int rapl_read_pl_data(struct rapl_domain *rd, int pl, enum pl_prims pl_prim, bool xlate, u64 *data) { - enum rapl_primitives prim = get_pl_prim(pl, pl_prim); + enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim); if (!is_pl_valid(rd, pl)) return -EINVAL; @@ -793,12 +805,12 @@ static int rapl_write_pl_data(struct rapl_domain *rd, int pl, enum pl_prims pl_prim, unsigned long long value) { - enum rapl_primitives prim = get_pl_prim(pl, pl_prim); + enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim); if (!is_pl_valid(rd, pl)) return -EINVAL; - if (rd->state & DOMAIN_STATE_BIOS_LOCKED) { + if (rd->rpl[pl].locked) { pr_warn("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]); return -EACCES; } @@ -1305,17 +1317,15 @@ static void rapl_detect_powerlimit(struct rapl_domain *rd) u64 val64; int i; - /* check if the domain is locked by BIOS, ignore if MSR doesn't exist */ - if (!rapl_read_data_raw(rd, FW_LOCK, false, &val64)) { - if (val64) { - pr_info("RAPL %s domain %s locked by BIOS\n", - rd->rp->name, rd->name); - rd->state |= DOMAIN_STATE_BIOS_LOCKED; - } - } - - /* check if power limit exists, otherwise domain is monitoring only */ for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) { + if (!rapl_read_pl_data(rd, i, PL_LOCK, false, &val64)) { + if (val64) { + rd->rpl[i].locked = true; + pr_info("%s:%s:%s locked by BIOS\n", + rd->rp->name, rd->name, pl_names[i]); + } + } + if (rapl_read_pl_data(rd, i, PL_ENABLE, false, &val64)) rd->rpl[i].name = NULL; } diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index df17f4e51dbf..d07b460bac3b 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -42,6 +42,7 @@ enum rapl_primitives { POWER_LIMIT4, ENERGY_COUNTER, FW_LOCK, + FW_HIGH_LOCK, PL1_ENABLE, /* power limit 1, aka long term */ PL1_CLAMP, /* allow frequency to go below OS request */ @@ -81,6 +82,7 @@ struct rapl_power_limit { struct powercap_zone_constraint *constraint; struct rapl_domain *domain; const char *name; + bool locked; u64 last_power_limit; }; From 693c1d7868cf710382c39c2d64cbb55e72b36d66 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:15 +0800 Subject: [PATCH 12/42] powercap: intel_rapl: Remove redundant cpu parameter For rapl_packages that rely on online CPUs to work, rp->lead_cpu always has a valid CPU id. Remove the redundant cpu parameter in rapl_check_domain(), rapl_detect_domains() and .check_unit() callbacks. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index d88008308d7a..fcb92f1f4bfc 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -178,7 +178,7 @@ static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim) struct rapl_defaults { u8 floor_freq_reg_addr; - int (*check_unit)(struct rapl_domain *rd, int cpu); + int (*check_unit)(struct rapl_domain *rd); void (*set_floor_freq)(struct rapl_domain *rd, bool mode); u64 (*compute_time_window)(struct rapl_domain *rd, u64 val, bool to_raw); @@ -828,16 +828,16 @@ static int rapl_write_pl_data(struct rapl_domain *rd, int pl, * power unit : microWatts : Represented in milliWatts by default * time unit : microseconds: Represented in seconds by default */ -static int rapl_check_unit_core(struct rapl_domain *rd, int cpu) +static int rapl_check_unit_core(struct rapl_domain *rd) { struct reg_action ra; u32 value; ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rd->rp->priv->read_raw(cpu, &ra)) { + if (rd->rp->priv->read_raw(rd->rp->lead_cpu, &ra)) { pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", - ra.reg, cpu); + ra.reg, rd->rp->lead_cpu); return -ENODEV; } @@ -856,16 +856,16 @@ static int rapl_check_unit_core(struct rapl_domain *rd, int cpu) return 0; } -static int rapl_check_unit_atom(struct rapl_domain *rd, int cpu) +static int rapl_check_unit_atom(struct rapl_domain *rd) { struct reg_action ra; u32 value; ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rd->rp->priv->read_raw(cpu, &ra)) { + if (rd->rp->priv->read_raw(rd->rp->lead_cpu, &ra)) { pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", - ra.reg, cpu); + ra.reg, rd->rp->lead_cpu); return -ENODEV; } @@ -1242,7 +1242,7 @@ err_cleanup: return ret; } -static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp) +static int rapl_check_domain(int domain, struct rapl_package *rp) { struct reg_action ra; @@ -1263,7 +1263,7 @@ static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp) */ ra.mask = ENERGY_STATUS_MASK; - if (rp->priv->read_raw(cpu, &ra) || !ra.value) + if (rp->priv->read_raw(rp->lead_cpu, &ra) || !ra.value) return -ENODEV; return 0; @@ -1292,7 +1292,7 @@ static int rapl_get_domain_unit(struct rapl_domain *rd) return -ENODEV; } - ret = defaults->check_unit(rd, rd->rp->lead_cpu); + ret = defaults->check_unit(rd); if (ret) return ret; @@ -1334,14 +1334,14 @@ static void rapl_detect_powerlimit(struct rapl_domain *rd) /* Detect active and valid domains for the given CPU, caller must * ensure the CPU belongs to the targeted package and CPU hotlug is disabled. */ -static int rapl_detect_domains(struct rapl_package *rp, int cpu) +static int rapl_detect_domains(struct rapl_package *rp) { struct rapl_domain *rd; int i; for (i = 0; i < RAPL_DOMAIN_MAX; i++) { /* use physical package id to read counters */ - if (!rapl_check_domain(cpu, i, rp)) { + if (!rapl_check_domain(i, rp)) { rp->domain_map |= 1 << i; pr_info("Found RAPL domain %s\n", rapl_domain_names[i]); } @@ -1445,7 +1445,7 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) topology_physical_package_id(cpu)); /* check if the package contains valid domains */ - if (rapl_detect_domains(rp, cpu)) { + if (rapl_detect_domains(rp)) { ret = -ENODEV; goto err_free_package; } From bf44b9011df3d6e34a23be77d86540553ba2bbe2 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:16 +0800 Subject: [PATCH 13/42] powercap: intel_rapl: Make cpu optional for rapl_package MSR RAPL Interface always removes a rapl_package when all the CPUs in that rapl_package are offlined. This is because it relies on an online CPU to access the MSR. But for RAPL Interface using MMIO registers, when all the cpus within the rapl_package are offlined, 1. the register can still be accessed 2. monitoring and setting the Power Pimits for the rapl_package is still meaningful because of uncore power. This means that, a valid rapl_package doesn't rely on one or more cpus being onlined. For this sense, make cpu optional for rapl_package. A rapl_package can be registered either using a CPU id to represent the physical package/die, or using the physical package id directly. Note that, the thermal throttling interrupt is not disabled via MSR_IA32_PACKAGE_THERM_INTERRUPT for such rapl_package at the moment. If it is still needed in the future, this can be achieved by selecting an onlined CPU using the physical package id. Note that, processor_thermal_rapl, the current MMIO RAPL Interface driver, can also be converted to register using a package id instead. But this is not done right now because processor_thermal_rapl driver works on single-package systems only, and offlining the only package will not happen. So keep the previous logic. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 84 +++++++++++-------- drivers/powercap/intel_rapl_msr.c | 6 +- .../int340x_thermal/processor_thermal_rapl.c | 6 +- include/linux/intel_rapl.h | 8 +- 4 files changed, 60 insertions(+), 44 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index fcb92f1f4bfc..d337270b65d4 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -532,6 +532,12 @@ static const struct powercap_zone_constraint_ops constraint_ops = { .get_name = get_constraint_name, }; +/* Return the id used for read_raw/write_raw callback */ +static int get_rid(struct rapl_package *rp) +{ + return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id; +} + /* called after domain detection and package level data are set */ static void rapl_init_domains(struct rapl_package *rp) { @@ -550,10 +556,12 @@ static void rapl_init_domains(struct rapl_package *rp) if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) { snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d", - topology_physical_package_id(rp->lead_cpu)); - } else + rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) : + rp->id); + } else { snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s", rapl_domain_names[i]); + } rd->id = i; @@ -725,7 +733,6 @@ static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim_fixed = prim_fixups(rd, prim); struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); struct reg_action ra; - int cpu; if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) return -EINVAL; @@ -734,8 +741,6 @@ static int rapl_read_data_raw(struct rapl_domain *rd, if (!ra.reg) return -EINVAL; - cpu = rd->rp->lead_cpu; - /* non-hardware data are collected by the polling thread */ if (rpi->flag & RAPL_PRIMITIVE_DERIVED) { *data = rd->rdd.primitives[prim]; @@ -744,8 +749,8 @@ static int rapl_read_data_raw(struct rapl_domain *rd, ra.mask = rpi->mask; - if (rd->rp->priv->read_raw(cpu, &ra)) { - pr_debug("failed to read reg 0x%llx on cpu %d\n", ra.reg, cpu); + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg, rd->rp->name, rd->name); return -EIO; } @@ -766,7 +771,6 @@ static int rapl_write_data_raw(struct rapl_domain *rd, { enum rapl_primitives prim_fixed = prim_fixups(rd, prim); struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); - int cpu; u64 bits; struct reg_action ra; int ret; @@ -774,7 +778,6 @@ static int rapl_write_data_raw(struct rapl_domain *rd, if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) return -EINVAL; - cpu = rd->rp->lead_cpu; bits = rapl_unit_xlate(rd, rpi->unit, value, 1); bits <<= rpi->shift; bits &= rpi->mask; @@ -785,7 +788,7 @@ static int rapl_write_data_raw(struct rapl_domain *rd, ra.mask = rpi->mask; ra.value = bits; - ret = rd->rp->priv->write_raw(cpu, &ra); + ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra); return ret; } @@ -835,9 +838,9 @@ static int rapl_check_unit_core(struct rapl_domain *rd) ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rd->rp->priv->read_raw(rd->rp->lead_cpu, &ra)) { - pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", - ra.reg, rd->rp->lead_cpu); + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n", + ra.reg, rd->rp->name, rd->name); return -ENODEV; } @@ -863,9 +866,9 @@ static int rapl_check_unit_atom(struct rapl_domain *rd) ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rd->rp->priv->read_raw(rd->rp->lead_cpu, &ra)) { - pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", - ra.reg, rd->rp->lead_cpu); + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n", + ra.reg, rd->rp->name, rd->name); return -ENODEV; } @@ -911,6 +914,9 @@ static void power_limit_irq_save_cpu(void *info) static void package_power_limit_irq_save(struct rapl_package *rp) { + if (rp->lead_cpu < 0) + return; + if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) return; @@ -925,6 +931,9 @@ static void package_power_limit_irq_restore(struct rapl_package *rp) { u32 l, h; + if (rp->lead_cpu < 0) + return; + if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) return; @@ -1263,7 +1272,7 @@ static int rapl_check_domain(int domain, struct rapl_package *rp) */ ra.mask = ENERGY_STATUS_MASK; - if (rp->priv->read_raw(rp->lead_cpu, &ra) || !ra.value) + if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value) return -ENODEV; return 0; @@ -1401,13 +1410,18 @@ void rapl_remove_package(struct rapl_package *rp) EXPORT_SYMBOL_GPL(rapl_remove_package); /* caller to ensure CPU hotplug lock is held */ -struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv) +struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) { - int id = topology_logical_die_id(cpu); struct rapl_package *rp; + int uid; + + if (id_is_cpu) + uid = topology_logical_die_id(id); + else + uid = id; list_for_each_entry(rp, &rapl_packages, plist) { - if (rp->id == id + if (rp->id == uid && rp->priv->control_type == priv->control_type) return rp; } @@ -1417,9 +1431,8 @@ struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv EXPORT_SYMBOL_GPL(rapl_find_package_domain); /* called from CPU hotplug notifier, hotplug lock held */ -struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) +struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) { - int id = topology_logical_die_id(cpu); struct rapl_package *rp; int ret; @@ -1427,23 +1440,26 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) if (!rp) return ERR_PTR(-ENOMEM); - /* add the new package to the list */ - rp->id = id; - rp->lead_cpu = cpu; - rp->priv = priv; + if (id_is_cpu) { + rp->id = topology_logical_die_id(id); + rp->lead_cpu = id; + if (topology_max_die_per_package() > 1) + snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d", + topology_physical_package_id(id), topology_die_id(id)); + else + snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", + topology_physical_package_id(id)); + } else { + rp->id = id; + rp->lead_cpu = -1; + snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id); + } + rp->priv = priv; ret = rapl_config(rp); if (ret) goto err_free_package; - if (topology_max_die_per_package() > 1) - snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, - "package-%d-die-%d", - topology_physical_package_id(cpu), topology_die_id(cpu)); - else - snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", - topology_physical_package_id(cpu)); - /* check if the package contains valid domains */ if (rapl_detect_domains(rp)) { ret = -ENODEV; diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 6fe5e556aa51..fff1a49d3b6c 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -68,9 +68,9 @@ static int rapl_cpu_online(unsigned int cpu) { struct rapl_package *rp; - rp = rapl_find_package_domain(cpu, rapl_msr_priv); + rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); if (!rp) { - rp = rapl_add_package(cpu, rapl_msr_priv); + rp = rapl_add_package(cpu, rapl_msr_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -83,7 +83,7 @@ static int rapl_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, rapl_msr_priv); + rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); if (!rp) return 0; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c index e070239106f5..140fb85cfa7b 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c @@ -27,9 +27,9 @@ static int rapl_mmio_cpu_online(unsigned int cpu) if (topology_physical_package_id(cpu)) return 0; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv); + rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); if (!rp) { - rp = rapl_add_package(cpu, &rapl_mmio_priv); + rp = rapl_add_package(cpu, &rapl_mmio_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -42,7 +42,7 @@ static int rapl_mmio_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv); + rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); if (!rp) return 0; diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index d07b460bac3b..51509f35027b 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -135,8 +135,8 @@ struct rapl_if_priv { u64 reg_unit; u64 regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; int limits[RAPL_DOMAIN_MAX]; - int (*read_raw)(int cpu, struct reg_action *ra); - int (*write_raw)(int cpu, struct reg_action *ra); + int (*read_raw)(int id, struct reg_action *ra); + int (*write_raw)(int id, struct reg_action *ra); void *defaults; void *rpi; }; @@ -161,8 +161,8 @@ struct rapl_package { struct rapl_if_priv *priv; }; -struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv); -struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv); +struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu); +struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu); void rapl_remove_package(struct rapl_package *rp); #endif /* __INTEL_RAPL_H__ */ From b4288ce788aaf160f2a706672af2eaef417bb057 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:17 +0800 Subject: [PATCH 14/42] powercap: intel_rapl: Introduce RAPL I/F type Different RAPL Interfaces may have different primitive information and rapl_defaults calls. To better distinguish this difference in the RAPL framework code, introduce a new enum to represent different types of RAPL Interfaces. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 18 +++++++++++++----- drivers/powercap/intel_rapl_msr.c | 2 ++ .../int340x_thermal/processor_thermal_rapl.c | 1 + include/linux/intel_rapl.h | 6 ++++++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index d337270b65d4..b70d6915a6ba 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -186,7 +186,7 @@ struct rapl_defaults { unsigned int psys_domain_energy_unit; bool spr_psys_bits; }; -static struct rapl_defaults *rapl_defaults; +static struct rapl_defaults *defaults_msr; static struct rapl_defaults *get_defaults(struct rapl_package *rp) { @@ -610,7 +610,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, return div64_u64(value, scale); } -static struct rapl_primitive_info rpi_default[NR_RAPL_PRIMITIVES] = { +static struct rapl_primitive_info rpi_msr[NR_RAPL_PRIMITIVES] = { /* name, mask, shift, msr index, unit divisor */ [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), @@ -679,8 +679,16 @@ static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim) static int rapl_config(struct rapl_package *rp) { - rp->priv->defaults = (void *)rapl_defaults; - rp->priv->rpi = (void *)rpi_default; + switch (rp->priv->type) { + /* MMIO I/F shares the same register layout as MSR registers */ + case RAPL_IF_MMIO: + case RAPL_IF_MSR: + rp->priv->defaults = (void *)defaults_msr; + rp->priv->rpi = (void *)rpi_msr; + break; + default: + return -EINVAL; + } return 0; } @@ -1546,7 +1554,7 @@ static int __init rapl_init(void) id = x86_match_cpu(rapl_ids); if (id) { - rapl_defaults = (struct rapl_defaults *)id->driver_data; + defaults_msr = (struct rapl_defaults *)id->driver_data; rapl_msr_platdev = platform_device_alloc("intel_rapl_msr", 0); if (!rapl_msr_platdev) diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index fff1a49d3b6c..cff5c6e8d570 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -34,6 +34,7 @@ static struct rapl_if_priv *rapl_msr_priv; static struct rapl_if_priv rapl_msr_priv_intel = { + .type = RAPL_IF_MSR, .reg_unit = MSR_RAPL_POWER_UNIT, .regs[RAPL_DOMAIN_PACKAGE] = { MSR_PKG_POWER_LIMIT, MSR_PKG_ENERGY_STATUS, MSR_PKG_PERF_STATUS, 0, MSR_PKG_POWER_INFO }, @@ -50,6 +51,7 @@ static struct rapl_if_priv rapl_msr_priv_intel = { }; static struct rapl_if_priv rapl_msr_priv_amd = { + .type = RAPL_IF_MSR, .reg_unit = MSR_AMD_RAPL_POWER_UNIT, .regs[RAPL_DOMAIN_PACKAGE] = { 0, MSR_AMD_PKG_ENERGY_STATUS, 0, 0, 0 }, diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c index 140fb85cfa7b..013f1633f082 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c @@ -97,6 +97,7 @@ int proc_thermal_rapl_add(struct pci_dev *pdev, struct proc_thermal_device *proc rapl_regs->regs[domain][reg]; rapl_mmio_priv.limits[domain] = rapl_regs->limits[domain]; } + rapl_mmio_priv.type = RAPL_IF_MMIO; rapl_mmio_priv.reg_unit = (u64)proc_priv->mmio_base + rapl_regs->reg_unit; rapl_mmio_priv.read_raw = rapl_mmio_read_raw; diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 51509f35027b..65f358b64096 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -14,6 +14,11 @@ #include #include +enum rapl_if_type { + RAPL_IF_MSR, /* RAPL I/F using MSR registers */ + RAPL_IF_MMIO, /* RAPL I/F using MMIO registers */ +}; + enum rapl_domain_type { RAPL_DOMAIN_PACKAGE, /* entire package/socket */ RAPL_DOMAIN_PP0, /* core power plane */ @@ -130,6 +135,7 @@ struct reg_action { * @rpi: internal pointer to interface primitive info */ struct rapl_if_priv { + enum rapl_if_type type; struct powercap_control_type *control_type; enum cpuhp_state pcap_rapl_online; u64 reg_unit; From e12dee18b89f1b0d4fc070eda4843f9d806645ca Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:18 +0800 Subject: [PATCH 15/42] powercap: intel_rapl: Introduce core support for TPMI interface Compared with existing RAPL MSR/MMIO Interface, the RAPL TPMI Interface 1. has per Power Limit register, thus has per Power Limit Lock and Enable bit. 2. doesn't have Power Limit Clamp bit. 3. the Power Limit Lock and Enable bits have different bit offsets. These mean RAPL TPMI Interface needs its own primitive information. RAPL TPMI Interface also has per domain unit register but with a different register layout. This requires a TPMI specific rapl_defaults call to decode the unit register. Introduce the RAPL core support for TPMI Interface. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 119 ++++++++++++++++++++++++++- include/linux/intel_rapl.h | 5 ++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index b70d6915a6ba..4e646e5e48f6 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -75,6 +75,15 @@ #define PSYS_TIME_WINDOW1_MASK (0x7FULL<<19) #define PSYS_TIME_WINDOW2_MASK (0x7FULL<<51) +/* bitmasks for RAPL TPMI, used by primitive access functions */ +#define TPMI_POWER_LIMIT_MASK 0x3FFFF +#define TPMI_POWER_LIMIT_ENABLE BIT_ULL(62) +#define TPMI_TIME_WINDOW_MASK (0x7FULL<<18) +#define TPMI_INFO_SPEC_MASK 0x3FFFF +#define TPMI_INFO_MIN_MASK (0x3FFFFULL << 18) +#define TPMI_INFO_MAX_MASK (0x3FFFFULL << 36) +#define TPMI_INFO_MAX_TIME_WIN_MASK (0x7FULL << 54) + /* Non HW constants */ #define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */ #define RAPL_PRIMITIVE_DUMMY BIT(2) @@ -119,6 +128,19 @@ static bool is_pl_valid(struct rapl_domain *rd, int pl) static int get_pl_lock_prim(struct rapl_domain *rd, int pl) { + if (rd->rp->priv->type == RAPL_IF_TPMI) { + if (pl == POWER_LIMIT1) + return PL1_LOCK; + if (pl == POWER_LIMIT2) + return PL2_LOCK; + if (pl == POWER_LIMIT4) + return PL4_LOCK; + } + + /* MSR/MMIO Interface doesn't have Lock bit for PL4 */ + if (pl == POWER_LIMIT4) + return -EINVAL; + /* * Power Limit register that supports two power limits has a different * bit position for the Lock bit. @@ -134,7 +156,7 @@ static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim) case POWER_LIMIT1: if (prim == PL_ENABLE) return PL1_ENABLE; - if (prim == PL_CLAMP) + if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) return PL1_CLAMP; if (prim == PL_LIMIT) return POWER_LIMIT1; @@ -148,7 +170,7 @@ static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim) case POWER_LIMIT2: if (prim == PL_ENABLE) return PL2_ENABLE; - if (prim == PL_CLAMP) + if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) return PL2_CLAMP; if (prim == PL_LIMIT) return POWER_LIMIT2; @@ -167,6 +189,8 @@ static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim) /* PL4 would be around two times PL2, use same prim as PL2. */ if (prim == PL_MAX_POWER) return MAX_POWER; + if (prim == PL_LOCK) + return get_pl_lock_prim(rd, pl); return -EINVAL; default: return -EINVAL; @@ -187,6 +211,7 @@ struct rapl_defaults { bool spr_psys_bits; }; static struct rapl_defaults *defaults_msr; +static const struct rapl_defaults defaults_tpmi; static struct rapl_defaults *get_defaults(struct rapl_package *rp) { @@ -610,6 +635,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, return div64_u64(value, scale); } +/* RAPL primitives for MSR and MMIO I/F */ static struct rapl_primitive_info rpi_msr[NR_RAPL_PRIMITIVES] = { /* name, mask, shift, msr index, unit divisor */ [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0, @@ -667,6 +693,48 @@ static struct rapl_primitive_info rpi_msr[NR_RAPL_PRIMITIVES] = { RAPL_PRIMITIVE_DERIVED), }; +/* RAPL primitives for TPMI I/F */ +static struct rapl_primitive_info rpi_tpmi[NR_RAPL_PRIMITIVES] = { + /* name, mask, shift, msr index, unit divisor */ + [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, TPMI_POWER_LIMIT_MASK, 0, + RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), + [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, TPMI_POWER_LIMIT_MASK, 0, + RAPL_DOMAIN_REG_PL2, POWER_UNIT, 0), + [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, TPMI_POWER_LIMIT_MASK, 0, + RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0), + [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0, + RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0), + [PL1_LOCK] = PRIMITIVE_INFO_INIT(PL1_LOCK, POWER_HIGH_LOCK, 63, + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), + [PL2_LOCK] = PRIMITIVE_INFO_INIT(PL2_LOCK, POWER_HIGH_LOCK, 63, + RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0), + [PL4_LOCK] = PRIMITIVE_INFO_INIT(PL4_LOCK, POWER_HIGH_LOCK, 63, + RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0), + [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62, + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), + [PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62, + RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0), + [PL4_ENABLE] = PRIMITIVE_INFO_INIT(PL4_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62, + RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0), + [TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TPMI_TIME_WINDOW_MASK, 18, + RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), + [TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TPMI_TIME_WINDOW_MASK, 18, + RAPL_DOMAIN_REG_PL2, TIME_UNIT, 0), + [THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, TPMI_INFO_SPEC_MASK, 0, + RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), + [MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, TPMI_INFO_MAX_MASK, 36, + RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), + [MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, TPMI_INFO_MIN_MASK, 18, + RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), + [MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, TPMI_INFO_MAX_TIME_WIN_MASK, 54, + RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0), + [THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0, + RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0), + /* non-hardware */ + [AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, + POWER_UNIT, RAPL_PRIMITIVE_DERIVED), +}; + static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim) { struct rapl_primitive_info *rpi = rp->priv->rpi; @@ -686,6 +754,10 @@ static int rapl_config(struct rapl_package *rp) rp->priv->defaults = (void *)defaults_msr; rp->priv->rpi = (void *)rpi_msr; break; + case RAPL_IF_TPMI: + rp->priv->defaults = (void *)&defaults_tpmi; + rp->priv->rpi = (void *)rpi_tpmi; + break; default: return -EINVAL; } @@ -1046,6 +1118,49 @@ static u64 rapl_compute_time_window_atom(struct rapl_domain *rd, u64 value, return value; } +/* TPMI Unit register has different layout */ +#define TPMI_POWER_UNIT_OFFSET POWER_UNIT_OFFSET +#define TPMI_POWER_UNIT_MASK POWER_UNIT_MASK +#define TPMI_ENERGY_UNIT_OFFSET 0x06 +#define TPMI_ENERGY_UNIT_MASK 0x7C0 +#define TPMI_TIME_UNIT_OFFSET 0x0C +#define TPMI_TIME_UNIT_MASK 0xF000 + +static int rapl_check_unit_tpmi(struct rapl_domain *rd) +{ + struct reg_action ra; + u32 value; + + ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; + ra.mask = ~0; + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n", + ra.reg, rd->rp->name, rd->name); + return -ENODEV; + } + + value = (ra.value & TPMI_ENERGY_UNIT_MASK) >> TPMI_ENERGY_UNIT_OFFSET; + rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); + + value = (ra.value & TPMI_POWER_UNIT_MASK) >> TPMI_POWER_UNIT_OFFSET; + rd->power_unit = 1000000 / (1 << value); + + value = (ra.value & TPMI_TIME_UNIT_MASK) >> TPMI_TIME_UNIT_OFFSET; + rd->time_unit = 1000000 / (1 << value); + + pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n", + rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); + + return 0; +} + +static const struct rapl_defaults defaults_tpmi = { + .check_unit = rapl_check_unit_tpmi, + /* Reuse existing logic, ignore the PL_CLAMP failures and enable all Power Limits */ + .set_floor_freq = set_floor_freq_default, + .compute_time_window = rapl_compute_time_window_core, +}; + static const struct rapl_defaults rapl_defaults_core = { .floor_freq_reg_addr = 0, .check_unit = rapl_check_unit_core, diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 65f358b64096..e6936cb25047 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -17,6 +17,7 @@ enum rapl_if_type { RAPL_IF_MSR, /* RAPL I/F using MSR registers */ RAPL_IF_MMIO, /* RAPL I/F using MMIO registers */ + RAPL_IF_TPMI, /* RAPL I/F using TPMI registers */ }; enum rapl_domain_type { @@ -36,6 +37,7 @@ enum rapl_domain_reg_id { RAPL_DOMAIN_REG_INFO, RAPL_DOMAIN_REG_PL4, RAPL_DOMAIN_REG_UNIT, + RAPL_DOMAIN_REG_PL2, RAPL_DOMAIN_REG_MAX, }; @@ -48,6 +50,9 @@ enum rapl_primitives { ENERGY_COUNTER, FW_LOCK, FW_HIGH_LOCK, + PL1_LOCK, + PL2_LOCK, + PL4_LOCK, PL1_ENABLE, /* power limit 1, aka long term */ PL1_CLAMP, /* allow frequency to go below OS request */ From 9eef7f9da928c54149199e7b3215b82c2d595ccd Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:19 +0800 Subject: [PATCH 16/42] powercap: intel_rapl: Introduce RAPL TPMI interface driver The TPMI (Topology Aware Register and PM Capsule Interface) provides a flexible, extendable and PCIe enumerable MMIO interface for PM features. Intel RAPL (Running Average Power Limit) is one of the features that benefit from this. Using TPMI Interface has advantage over traditional MSR (Model Specific Register) interface, where a thread needs to be scheduled on the target CPU to read or write. Also the RAPL features vary between CPU models, and hence lot of model specific code. Here TPMI provides an architectural interface by providing hierarchical tables and fields, which will not need any model specific implementation. TPMI interface uses a PCI VSEC structure to expose the location of MMIO interface for PM feature enumeration and control. The Intel VSEC driver parses VSEC structures present in the PCI configuration space of the given device and creates an auxiliary device object for each of them. In particular, it creates an auxiliary device object representing TPMI that can be bound to by an auxiliary driver. Then the TPMI enumeration driver binds to the TPMI auxiliary device object created by the Intel VSEC driver, parses the PM Feature Structure (PFS) present in the TPMI MMIO region and creates device nodes for PM features described in the PFS. This RAPL TPMI Interface driver binds the RAPL auxiliary device created by the TPMI enumeration driver and expose the RAPL control to userspace via powercap sysfs class. RAPL TPMI details are published in the following document: https://github.com/intel/tpmi_power_management/blob/main/RAPL_TPMI_public_disclosure_FINAL.docx Note, for now, the RAPL TPMI Interface and RAPL MSR Interface cannot co-exists on the same platform (RAPL TPMI Interface is not supported on any platforms in the CPU model list for RAPL MSR Interface). Thus register the RAPL TPMI powercap control type with name "intel-rapl", the same as RAPL MSR Interface, so that it is transparent to userspace. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- drivers/powercap/Kconfig | 14 ++ drivers/powercap/Makefile | 1 + drivers/powercap/intel_rapl_tpmi.c | 325 +++++++++++++++++++++++++++++ 3 files changed, 340 insertions(+) create mode 100644 drivers/powercap/intel_rapl_tpmi.c diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index 90d33cd1b670..e71399804c14 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -33,6 +33,20 @@ config INTEL_RAPL controller, CPU core (Power Plane 0), graphics uncore (Power Plane 1), etc. +config INTEL_RAPL_TPMI + tristate "Intel RAPL Support via TPMI Interface" + depends on X86 + depends on INTEL_TPMI + select INTEL_RAPL_CORE + help + This enables support for the Intel Running Average Power Limit (RAPL) + technology via TPMI interface, which allows power limits to be enforced + and monitored. + + In RAPL, the platform level settings are divided into domains for + fine grained control. These domains include processor package, DRAM + controller, platform, etc. + config IDLE_INJECT bool "Idle injection framework" depends on CPU_IDLE diff --git a/drivers/powercap/Makefile b/drivers/powercap/Makefile index 4474201b4aa7..5ab0dce565b9 100644 --- a/drivers/powercap/Makefile +++ b/drivers/powercap/Makefile @@ -5,5 +5,6 @@ obj-$(CONFIG_DTPM_DEVFREQ) += dtpm_devfreq.o obj-$(CONFIG_POWERCAP) += powercap_sys.o obj-$(CONFIG_INTEL_RAPL_CORE) += intel_rapl_common.o obj-$(CONFIG_INTEL_RAPL) += intel_rapl_msr.o +obj-$(CONFIG_INTEL_RAPL_TPMI) += intel_rapl_tpmi.o obj-$(CONFIG_IDLE_INJECT) += idle_inject.o obj-$(CONFIG_ARM_SCMI_POWERCAP) += arm_scmi_powercap.o diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c new file mode 100644 index 000000000000..c016127b3497 --- /dev/null +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * intel_rapl_tpmi: Intel RAPL driver via TPMI interface + * + * Copyright (c) 2023, Intel Corporation. + * All Rights Reserved. + * + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#define TPMI_RAPL_VERSION 1 + +/* 1 header + 10 registers + 5 reserved. 8 bytes for each. */ +#define TPMI_RAPL_DOMAIN_SIZE 128 + +enum tpmi_rapl_domain_type { + TPMI_RAPL_DOMAIN_INVALID, + TPMI_RAPL_DOMAIN_SYSTEM, + TPMI_RAPL_DOMAIN_PACKAGE, + TPMI_RAPL_DOMAIN_RESERVED, + TPMI_RAPL_DOMAIN_MEMORY, + TPMI_RAPL_DOMAIN_MAX, +}; + +enum tpmi_rapl_register { + TPMI_RAPL_REG_HEADER, + TPMI_RAPL_REG_UNIT, + TPMI_RAPL_REG_PL1, + TPMI_RAPL_REG_PL2, + TPMI_RAPL_REG_PL3, + TPMI_RAPL_REG_PL4, + TPMI_RAPL_REG_RESERVED, + TPMI_RAPL_REG_ENERGY_STATUS, + TPMI_RAPL_REG_PERF_STATUS, + TPMI_RAPL_REG_POWER_INFO, + TPMI_RAPL_REG_INTERRUPT, + TPMI_RAPL_REG_MAX = 15, +}; + +struct tpmi_rapl_package { + struct rapl_if_priv priv; + struct intel_tpmi_plat_info *tpmi_info; + struct rapl_package *rp; + void __iomem *base; + struct list_head node; +}; + +static LIST_HEAD(tpmi_rapl_packages); +static DEFINE_MUTEX(tpmi_rapl_lock); + +static struct powercap_control_type *tpmi_control_type; + +static int tpmi_rapl_read_raw(int id, struct reg_action *ra) +{ + if (!ra->reg) + return -EINVAL; + + ra->value = readq((void __iomem *)ra->reg); + + ra->value &= ra->mask; + return 0; +} + +static int tpmi_rapl_write_raw(int id, struct reg_action *ra) +{ + u64 val; + + if (!ra->reg) + return -EINVAL; + + val = readq((void __iomem *)ra->reg); + + val &= ~ra->mask; + val |= ra->value; + + writeq(val, (void __iomem *)ra->reg); + return 0; +} + +static struct tpmi_rapl_package *trp_alloc(int pkg_id) +{ + struct tpmi_rapl_package *trp; + int ret; + + mutex_lock(&tpmi_rapl_lock); + + if (list_empty(&tpmi_rapl_packages)) { + tpmi_control_type = powercap_register_control_type(NULL, "intel-rapl", NULL); + if (IS_ERR(tpmi_control_type)) { + ret = PTR_ERR(tpmi_control_type); + goto err_unlock; + } + } + + trp = kzalloc(sizeof(*trp), GFP_KERNEL); + if (!trp) { + ret = -ENOMEM; + goto err_del_powercap; + } + + list_add(&trp->node, &tpmi_rapl_packages); + + mutex_unlock(&tpmi_rapl_lock); + return trp; + +err_del_powercap: + if (list_empty(&tpmi_rapl_packages)) + powercap_unregister_control_type(tpmi_control_type); +err_unlock: + mutex_unlock(&tpmi_rapl_lock); + return ERR_PTR(ret); +} + +static void trp_release(struct tpmi_rapl_package *trp) +{ + mutex_lock(&tpmi_rapl_lock); + list_del(&trp->node); + + if (list_empty(&tpmi_rapl_packages)) + powercap_unregister_control_type(tpmi_control_type); + + kfree(trp); + mutex_unlock(&tpmi_rapl_lock); +} + +static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) +{ + u8 tpmi_domain_version; + enum rapl_domain_type domain_type; + enum tpmi_rapl_domain_type tpmi_domain_type; + enum tpmi_rapl_register reg_index; + enum rapl_domain_reg_id reg_id; + int tpmi_domain_size, tpmi_domain_flags; + u64 *tpmi_rapl_regs = trp->base + offset; + u64 tpmi_domain_header = readq((void __iomem *)tpmi_rapl_regs); + + /* Domain Parent bits are ignored for now */ + tpmi_domain_version = tpmi_domain_header & 0xff; + tpmi_domain_type = tpmi_domain_header >> 8 & 0xff; + tpmi_domain_size = tpmi_domain_header >> 16 & 0xff; + tpmi_domain_flags = tpmi_domain_header >> 32 & 0xffff; + + if (tpmi_domain_version != TPMI_RAPL_VERSION) { + pr_warn(FW_BUG "Unsupported version:%d\n", tpmi_domain_version); + return -ENODEV; + } + + /* Domain size: in unit of 128 Bytes */ + if (tpmi_domain_size != 1) { + pr_warn(FW_BUG "Invalid Domain size %d\n", tpmi_domain_size); + return -EINVAL; + } + + /* Unit register and Energy Status register are mandatory for each domain */ + if (!(tpmi_domain_flags & BIT(TPMI_RAPL_REG_UNIT)) || + !(tpmi_domain_flags & BIT(TPMI_RAPL_REG_ENERGY_STATUS))) { + pr_warn(FW_BUG "Invalid Domain flag 0x%x\n", tpmi_domain_flags); + return -EINVAL; + } + + switch (tpmi_domain_type) { + case TPMI_RAPL_DOMAIN_PACKAGE: + domain_type = RAPL_DOMAIN_PACKAGE; + break; + case TPMI_RAPL_DOMAIN_SYSTEM: + domain_type = RAPL_DOMAIN_PLATFORM; + break; + case TPMI_RAPL_DOMAIN_MEMORY: + domain_type = RAPL_DOMAIN_DRAM; + break; + default: + pr_warn(FW_BUG "Unsupported Domain type %d\n", tpmi_domain_type); + return -EINVAL; + } + + if (trp->priv.regs[domain_type][RAPL_DOMAIN_REG_UNIT]) { + pr_warn(FW_BUG "Duplicate Domain type %d\n", tpmi_domain_type); + return -EINVAL; + } + + reg_index = TPMI_RAPL_REG_HEADER; + while (++reg_index != TPMI_RAPL_REG_MAX) { + if (!(tpmi_domain_flags & BIT(reg_index))) + continue; + + switch (reg_index) { + case TPMI_RAPL_REG_UNIT: + reg_id = RAPL_DOMAIN_REG_UNIT; + break; + case TPMI_RAPL_REG_PL1: + reg_id = RAPL_DOMAIN_REG_LIMIT; + trp->priv.limits[domain_type] |= BIT(POWER_LIMIT1); + break; + case TPMI_RAPL_REG_PL2: + reg_id = RAPL_DOMAIN_REG_PL2; + trp->priv.limits[domain_type] |= BIT(POWER_LIMIT2); + break; + case TPMI_RAPL_REG_PL4: + reg_id = RAPL_DOMAIN_REG_PL4; + trp->priv.limits[domain_type] |= BIT(POWER_LIMIT4); + break; + case TPMI_RAPL_REG_ENERGY_STATUS: + reg_id = RAPL_DOMAIN_REG_STATUS; + break; + case TPMI_RAPL_REG_PERF_STATUS: + reg_id = RAPL_DOMAIN_REG_PERF; + break; + case TPMI_RAPL_REG_POWER_INFO: + reg_id = RAPL_DOMAIN_REG_INFO; + break; + default: + continue; + } + trp->priv.regs[domain_type][reg_id] = (u64)&tpmi_rapl_regs[reg_index]; + } + + return 0; +} + +static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev, + const struct auxiliary_device_id *id) +{ + struct tpmi_rapl_package *trp; + struct intel_tpmi_plat_info *info; + struct resource *res; + u32 offset; + int ret; + + info = tpmi_get_platform_data(auxdev); + if (!info) + return -ENODEV; + + trp = trp_alloc(info->package_id); + if (IS_ERR(trp)) + return PTR_ERR(trp); + + if (tpmi_get_resource_count(auxdev) > 1) { + dev_err(&auxdev->dev, "does not support multiple resources\n"); + ret = -EINVAL; + goto err; + } + + res = tpmi_get_resource_at_index(auxdev, 0); + if (!res) { + dev_err(&auxdev->dev, "can't fetch device resource info\n"); + ret = -EIO; + goto err; + } + + trp->base = devm_ioremap_resource(&auxdev->dev, res); + if (!trp->base) { + ret = -ENOMEM; + goto err; + } + + for (offset = 0; offset < resource_size(res); offset += TPMI_RAPL_DOMAIN_SIZE) { + ret = parse_one_domain(trp, offset); + if (ret) + goto err; + } + + trp->tpmi_info = info; + trp->priv.type = RAPL_IF_TPMI; + trp->priv.read_raw = tpmi_rapl_read_raw; + trp->priv.write_raw = tpmi_rapl_write_raw; + trp->priv.control_type = tpmi_control_type; + + /* RAPL TPMI I/F is per physical package */ + trp->rp = rapl_find_package_domain(info->package_id, &trp->priv, false); + if (trp->rp) { + dev_err(&auxdev->dev, "Domain for Package%d already exists\n", info->package_id); + ret = -EEXIST; + goto err; + } + + trp->rp = rapl_add_package(info->package_id, &trp->priv, false); + if (IS_ERR(trp->rp)) { + dev_err(&auxdev->dev, "Failed to add RAPL Domain for Package%d, %ld\n", + info->package_id, PTR_ERR(trp->rp)); + ret = PTR_ERR(trp->rp); + goto err; + } + + auxiliary_set_drvdata(auxdev, trp); + + return 0; +err: + trp_release(trp); + return ret; +} + +static void intel_rapl_tpmi_remove(struct auxiliary_device *auxdev) +{ + struct tpmi_rapl_package *trp = auxiliary_get_drvdata(auxdev); + + rapl_remove_package(trp->rp); + trp_release(trp); +} + +static const struct auxiliary_device_id intel_rapl_tpmi_ids[] = { + {.name = "intel_vsec.tpmi-rapl" }, + { } +}; + +MODULE_DEVICE_TABLE(auxiliary, intel_rapl_tpmi_ids); + +static struct auxiliary_driver intel_rapl_tpmi_driver = { + .probe = intel_rapl_tpmi_probe, + .remove = intel_rapl_tpmi_remove, + .id_table = intel_rapl_tpmi_ids, +}; + +module_auxiliary_driver(intel_rapl_tpmi_driver) + +MODULE_IMPORT_NS(INTEL_TPMI); + +MODULE_DESCRIPTION("Intel RAPL TPMI Driver"); +MODULE_LICENSE("GPL"); From 2e41e3ca4729455e002bcb585f0d3749ee66d572 Mon Sep 17 00:00:00 2001 From: Kai-Heng Feng Date: Tue, 2 May 2023 17:04:34 +0200 Subject: [PATCH 17/42] PM: suspend: Fix pm_suspend_target_state handling for !CONFIG_PM Move the pm_suspend_target_state definition for CONFIG_SUSPEND unset from the wakeup code into the headers so as to allow it to still be used elsewhere when CONFIG_SUSPEND is not set. Signed-off-by: Kai-Heng Feng [ rjw: Changelog and subject edits ] Signed-off-by: Rafael J. Wysocki --- drivers/base/power/wakeup.c | 5 ----- include/linux/suspend.h | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 7cc0c0cf8eaa..a917219feea6 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -19,11 +19,6 @@ #include "power.h" -#ifndef CONFIG_SUSPEND -suspend_state_t pm_suspend_target_state; -#define pm_suspend_target_state (PM_SUSPEND_ON) -#endif - #define list_for_each_entry_rcu_locked(pos, head, member) \ list_for_each_entry_rcu(pos, head, member, \ srcu_read_lock_held(&wakeup_srcu)) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index d0d4598a7b3f..474ecfbbaa62 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -202,6 +202,7 @@ struct platform_s2idle_ops { }; #ifdef CONFIG_SUSPEND +extern suspend_state_t pm_suspend_target_state; extern suspend_state_t mem_sleep_current; extern suspend_state_t mem_sleep_default; @@ -337,6 +338,8 @@ extern bool sync_on_suspend_enabled; #else /* !CONFIG_SUSPEND */ #define suspend_valid_only_mem NULL +#define pm_suspend_target_state (PM_SUSPEND_ON) + static inline void pm_suspend_clear_flags(void) {} static inline void pm_set_suspend_via_firmware(void) {} static inline void pm_set_resume_via_firmware(void) {} @@ -503,7 +506,6 @@ extern void pm_report_max_hw_sleep(u64 t); /* drivers/base/power/wakeup.c */ extern bool events_check_enabled; -extern suspend_state_t pm_suspend_target_state; extern bool pm_wakeup_pending(void); extern void pm_system_wakeup(void); From 847aea98e01cf084efcb84490b3060af343d1458 Mon Sep 17 00:00:00 2001 From: Wang Honghui Date: Thu, 4 May 2023 17:18:24 +0800 Subject: [PATCH 18/42] PM: hibernate: Correct spelling mistake in a comment Fix a typo in a comment in kernel/power/snapshot.c Signed-off-by: Wang Honghui [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- kernel/power/snapshot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index cd8b7b35f1e8..b27affb7503f 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -398,7 +398,7 @@ struct mem_zone_bm_rtree { unsigned int blocks; /* Number of Bitmap Blocks */ }; -/* strcut bm_position is used for browsing memory bitmaps */ +/* struct bm_position is used for browsing memory bitmaps */ struct bm_position { struct mem_zone_bm_rtree *zone; From ab23ed6e73ecd198bf577077677beaded0a9e718 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:10:58 +0200 Subject: [PATCH 19/42] PM: suspend: add a arch_resume_nosmt() prototype The arch_resume_nosmt() has a __weak definition, plus an x86 specific override, but no prototype that ensures the two have the same arguments. This causes a W=1 warning: arch/x86/power/hibernate.c:189:5: error: no previous prototype for 'arch_resume_nosmt' [-Werror=missing-prototypes] Add the prototype in linux/suspend.h, which is included in both places. Signed-off-by: Arnd Bergmann Signed-off-by: Rafael J. Wysocki --- include/linux/suspend.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 474ecfbbaa62..1a0426e6761c 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -471,6 +471,8 @@ static inline int hibernate_quiet_exec(int (*func)(void *data), void *data) { } #endif /* CONFIG_HIBERNATION */ +int arch_resume_nosmt(void); + #ifdef CONFIG_HIBERNATION_SNAPSHOT_DEV int is_hibernate_resume_dev(dev_t dev); #else From 4a3a2c32a5ee163bc8f195b04751f165aa4d9c83 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 8 May 2023 09:42:15 +0200 Subject: [PATCH 20/42] PM / devfreq: Reorder fields in 'struct devfreq_dev_status' Group some variables based on their sizes to reduce holes. On x86_64, this shrinks the size of 'struct devfreq_dev_status' from 72 to 64 bytes. This structure is used both to allocate static variables or is embedded in some other structures. In both cases, reducing its size is nice to have. Moreover, the whole structure now fits in a single cache line on x86_64. Finally, it makes the order of code match the order of the above kernel doc. Signed-off-by: Christophe JAILLET Acked-by: MyungJoo Ham Signed-off-by: Chanwoo Choi --- include/linux/devfreq.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h index 7fd704bb8f3d..d312ffbac4dd 100644 --- a/include/linux/devfreq.h +++ b/include/linux/devfreq.h @@ -108,7 +108,6 @@ struct devfreq_dev_profile { unsigned long initial_freq; unsigned int polling_ms; enum devfreq_timer timer; - bool is_cooling_device; int (*target)(struct device *dev, unsigned long *freq, u32 flags); int (*get_dev_status)(struct device *dev, @@ -118,6 +117,8 @@ struct devfreq_dev_profile { unsigned long *freq_table; unsigned int max_state; + + bool is_cooling_device; }; /** From ccb69e228ea48f8ea1e4a7dfeedf501329a9fdf4 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 May 2023 09:44:03 +0200 Subject: [PATCH 21/42] PM / devfreq: exynos: add Exynos PPMU as a soft module dependency Commit adf8238ef403 ("ARM: dts: exynos: move exynos-bus nodes out of soc in Exynos4412") changed the order of the exynos-bus nodes, what results in different probe order of the Exynos Bus devices. Although the driver properly handles the deferred probe and all devices seems to be finally properly registered, this change revealed some kind of a bug related to PPMU counters registration and passive governor operation. Usually in 1 of 10 boots this results in complete board freeze during loading of the kernel modules. To avoid that freeze, ensure that the Exynos PPMU driver is already loaded before the Exynos Bus driver starts probing. Signed-off-by: Marek Szyprowski Reviewed-by: Krzysztof Kozlowski Signed-off-by: Chanwoo Choi --- drivers/devfreq/exynos-bus.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 88414445adf3..245898f1a88e 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -518,6 +518,7 @@ static struct platform_driver exynos_bus_platdrv = { }; module_platform_driver(exynos_bus_platdrv); +MODULE_SOFTDEP("pre: exynos_ppmu"); MODULE_DESCRIPTION("Generic Exynos Bus frequency driver"); MODULE_AUTHOR("Chanwoo Choi "); MODULE_LICENSE("GPL v2"); From a83bfdca8b2098999e3edfb87e98925e019eb818 Mon Sep 17 00:00:00 2001 From: Sukrut Bellary Date: Thu, 18 May 2023 01:40:33 -0700 Subject: [PATCH 22/42] PM / devfreq: mtk-cci: Fix variable deferencing before NULL check smatch warning: drivers/devfreq/mtk-cci-devfreq.c:135 mtk_ccifreq_target() warn: variable dereferenced before check 'drv' (see line 130) This is based on static analysis only. Compilation tested. Signed-off-by: Sukrut Bellary Signed-off-by: Chanwoo Choi --- drivers/devfreq/mtk-cci-devfreq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/devfreq/mtk-cci-devfreq.c b/drivers/devfreq/mtk-cci-devfreq.c index e5458ada5197..6354622eda65 100644 --- a/drivers/devfreq/mtk-cci-devfreq.c +++ b/drivers/devfreq/mtk-cci-devfreq.c @@ -127,7 +127,7 @@ static int mtk_ccifreq_target(struct device *dev, unsigned long *freq, u32 flags) { struct mtk_ccifreq_drv *drv = dev_get_drvdata(dev); - struct clk *cci_pll = clk_get_parent(drv->cci_clk); + struct clk *cci_pll; struct dev_pm_opp *opp; unsigned long opp_rate; int voltage, pre_voltage, inter_voltage, target_voltage, ret; @@ -139,6 +139,7 @@ static int mtk_ccifreq_target(struct device *dev, unsigned long *freq, return 0; inter_voltage = drv->inter_voltage; + cci_pll = clk_get_parent(drv->cci_clk); opp_rate = *freq; opp = devfreq_recommended_opp(dev, &opp_rate, 1); From d05b5e0baf424c8c4b4709ac11f66ab726c8deaf Mon Sep 17 00:00:00 2001 From: Sumeet Pawnikar Date: Thu, 8 Jun 2023 08:00:06 +0530 Subject: [PATCH 23/42] powercap: RAPL: fix invalid initialization for pl4_supported field The current initialization of the struct x86_cpu_id via pl4_support_ids[] is partial and wrong. It is initializing "stepping" field with "X86_FEATURE_ANY" instead of "feature" field. Use X86_MATCH_INTEL_FAM6_MODEL macro instead of initializing each field of the struct x86_cpu_id for pl4_supported list of CPUs. This X86_MATCH_INTEL_FAM6_MODEL macro internally uses another macro X86_MATCH_VENDOR_FAM_MODEL_FEATURE for X86 based CPU matching with appropriate initialized values. Reported-by: Dave Hansen Link: https://lore.kernel.org/lkml/28ead36b-2d9e-1a36-6f4e-04684e420260@intel.com Fixes: eb52bc2ae5b8 ("powercap: RAPL: Add Power Limit4 support for Meteor Lake SoC") Fixes: b08b95cf30f5 ("powercap: RAPL: Add Power Limit4 support for Alder Lake-N and Raptor Lake-P") Fixes: 515755906921 ("powercap: RAPL: Add Power Limit4 support for RaptorLake") Fixes: 1cc5b9a411e4 ("powercap: Add Power Limit4 support for Alder Lake SoC") Fixes: 8365a898fe53 ("powercap: Add Power Limit4 support") Signed-off-by: Sumeet Pawnikar Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_msr.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index cff5c6e8d570..713cf1cdff29 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -139,14 +139,14 @@ static int rapl_msr_write_raw(int cpu, struct reg_action *ra) /* List of verified CPUs. */ static const struct x86_cpu_id pl4_support_ids[] = { - { X86_VENDOR_INTEL, 6, INTEL_FAM6_TIGERLAKE_L, X86_FEATURE_ANY }, - { X86_VENDOR_INTEL, 6, INTEL_FAM6_ALDERLAKE, X86_FEATURE_ANY }, - { X86_VENDOR_INTEL, 6, INTEL_FAM6_ALDERLAKE_L, X86_FEATURE_ANY }, - { X86_VENDOR_INTEL, 6, INTEL_FAM6_ALDERLAKE_N, X86_FEATURE_ANY }, - { X86_VENDOR_INTEL, 6, INTEL_FAM6_RAPTORLAKE, X86_FEATURE_ANY }, - { X86_VENDOR_INTEL, 6, INTEL_FAM6_RAPTORLAKE_P, X86_FEATURE_ANY }, - { X86_VENDOR_INTEL, 6, INTEL_FAM6_METEORLAKE, X86_FEATURE_ANY }, - { X86_VENDOR_INTEL, 6, INTEL_FAM6_METEORLAKE_L, X86_FEATURE_ANY }, + X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, NULL), + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, NULL), + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, NULL), + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, NULL), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, NULL), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, NULL), + X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, NULL), + X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, NULL), {} }; From 4658fe81b3f8afe8adf37734ec5fe595d90415c6 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Tue, 6 Jun 2023 22:00:00 +0800 Subject: [PATCH 24/42] powercap: RAPL: Fix CONFIG_IOSF_MBI dependency After commit 3382388d7148 ("intel_rapl: abstract RAPL common code"), accessing to IOSF_MBI interface is done in the RAPL common code. Thus it is the CONFIG_INTEL_RAPL_CORE that has dependency of CONFIG_IOSF_MBI, while CONFIG_INTEL_RAPL_MSR does not. This problem was not exposed previously because all the previous RAPL common code users, aka, the RAPL MSR and MMIO I/F drivers, have CONFIG_IOSF_MBI selected. Fix the CONFIG_IOSF_MBI dependency in RAPL code. This also fixes a build time failure when the RAPL TPMI I/F driver is introduced without selecting CONFIG_IOSF_MBI. x86_64-linux-ld: vmlinux.o: in function `set_floor_freq_atom': intel_rapl_common.c:(.text+0x2dac9b8): undefined reference to `iosf_mbi_write' x86_64-linux-ld: intel_rapl_common.c:(.text+0x2daca66): undefined reference to `iosf_mbi_read' Reference to iosf_mbi.h is also removed from the RAPL MSR I/F driver. Fixes: 3382388d7148 ("intel_rapl: abstract RAPL common code") Reported-by: Arnd Bergmann Link: https://lore.kernel.org/all/20230601213246.3271412-1-arnd@kernel.org Signed-off-by: Zhang Rui Signed-off-by: Rafael J. Wysocki --- drivers/powercap/Kconfig | 4 +++- drivers/powercap/intel_rapl_msr.c | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index e71399804c14..69ef8d081c98 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -18,10 +18,12 @@ if POWERCAP # Client driver configurations go here. config INTEL_RAPL_CORE tristate + depends on PCI + select IOSF_MBI config INTEL_RAPL tristate "Intel RAPL Support via MSR Interface" - depends on X86 && IOSF_MBI + depends on X86 && PCI select INTEL_RAPL_CORE help This enables support for the Intel Running Average Power Limit (RAPL) diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 713cf1cdff29..569e25eab1e1 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -22,7 +22,6 @@ #include #include -#include #include #include From 49776c712eb6ded12fcbb8cc915a498dbfb47311 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 6 Jun 2023 11:22:42 +0300 Subject: [PATCH 25/42] powercap: RAPL: Fix a NULL vs IS_ERR() bug The devm_ioremap_resource() function returns error pointers on error, it never returns NULL. Update the check accordingly. Fixes: 9eef7f9da928 ("powercap: intel_rapl: Introduce RAPL TPMI interface driver") Signed-off-by: Dan Carpenter Acked-by: Zhang Rui Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_tpmi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index c016127b3497..4f4f13ded225 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -255,8 +255,8 @@ static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev, } trp->base = devm_ioremap_resource(&auxdev->dev, res); - if (!trp->base) { - ret = -ENOMEM; + if (IS_ERR(trp->base)) { + ret = PTR_ERR(trp->base); goto err; } From cdb8c100d8a4b4e31c829724e40b4fdf32977cce Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 2 Jun 2023 02:30:22 -0500 Subject: [PATCH 26/42] include/linux/suspend.h: Only show pm_pr_dbg messages at suspend/resume All uses in the kernel are currently already oriented around suspend/resume. As some other parts of the kernel may also use these messages in functions that could also be used outside of suspend/resume, only enable in suspend/resume path. Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- include/linux/suspend.h | 8 +++++--- kernel/power/main.c | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 1a0426e6761c..74f406c53ac0 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -555,6 +555,7 @@ static inline void unlock_system_sleep(unsigned int flags) {} #ifdef CONFIG_PM_SLEEP_DEBUG extern bool pm_print_times_enabled; extern bool pm_debug_messages_on; +extern bool pm_debug_messages_should_print(void); static inline int pm_dyn_debug_messages_on(void) { #ifdef CONFIG_DYNAMIC_DEBUG @@ -568,14 +569,14 @@ static inline int pm_dyn_debug_messages_on(void) #endif #define __pm_pr_dbg(fmt, ...) \ do { \ - if (pm_debug_messages_on) \ + if (pm_debug_messages_should_print()) \ printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ else if (pm_dyn_debug_messages_on()) \ pr_debug(fmt, ##__VA_ARGS__); \ } while (0) #define __pm_deferred_pr_dbg(fmt, ...) \ do { \ - if (pm_debug_messages_on) \ + if (pm_debug_messages_should_print()) \ printk_deferred(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ } while (0) #else @@ -593,7 +594,8 @@ static inline int pm_dyn_debug_messages_on(void) /** * pm_pr_dbg - print pm sleep debug messages * - * If pm_debug_messages_on is enabled, print message. + * If pm_debug_messages_on is enabled and the system is entering/leaving + * suspend, print message. * If pm_debug_messages_on is disabled and CONFIG_DYNAMIC_DEBUG is enabled, * print message only from instances explicitly enabled on dynamic debug's * control. diff --git a/kernel/power/main.c b/kernel/power/main.c index 3113ec2f1db4..daa535012e51 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -556,6 +556,12 @@ power_attr_ro(pm_wakeup_irq); bool pm_debug_messages_on __read_mostly; +bool pm_debug_messages_should_print(void) +{ + return pm_debug_messages_on && pm_suspend_target_state != PM_SUSPEND_ON; +} +EXPORT_SYMBOL_GPL(pm_debug_messages_should_print); + static ssize_t pm_debug_messages_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { From f75400603f5fb7a031218c659852809378a77cb7 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 2 Jun 2023 02:30:23 -0500 Subject: [PATCH 27/42] ACPI: x86: Add pm_debug_messages for LPS0 _DSM state tracking Enabling debugging messages for the state requires turning on dynamic debugging for the file. To make it more accessible, use `pm_debug_messages` and clearer strings for what is happening. Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- drivers/acpi/x86/s2idle.c | 52 ++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index e499c60c4579..7681f6ecab67 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -59,6 +59,7 @@ static int lps0_dsm_func_mask; static guid_t lps0_dsm_guid_microsoft; static int lps0_dsm_func_mask_microsoft; +static int lps0_dsm_state; /* Device constraint entry structure */ struct lpi_device_info { @@ -320,6 +321,44 @@ static void lpi_check_constraints(void) } } +static bool acpi_s2idle_vendor_amd(void) +{ + return boot_cpu_data.x86_vendor == X86_VENDOR_AMD; +} + +static const char *acpi_sleep_dsm_state_to_str(unsigned int state) +{ + if (lps0_dsm_func_mask_microsoft || !acpi_s2idle_vendor_amd()) { + switch (state) { + case ACPI_LPS0_SCREEN_OFF: + return "screen off"; + case ACPI_LPS0_SCREEN_ON: + return "screen on"; + case ACPI_LPS0_ENTRY: + return "lps0 entry"; + case ACPI_LPS0_EXIT: + return "lps0 exit"; + case ACPI_LPS0_MS_ENTRY: + return "lps0 ms entry"; + case ACPI_LPS0_MS_EXIT: + return "lps0 ms exit"; + } + } else { + switch (state) { + case ACPI_LPS0_SCREEN_ON_AMD: + return "screen on"; + case ACPI_LPS0_SCREEN_OFF_AMD: + return "screen off"; + case ACPI_LPS0_ENTRY_AMD: + return "lps0 entry"; + case ACPI_LPS0_EXIT_AMD: + return "lps0 exit"; + } + } + + return "unknown"; +} + static void acpi_sleep_run_lps0_dsm(unsigned int func, unsigned int func_mask, guid_t dsm_guid) { union acpi_object *out_obj; @@ -331,14 +370,15 @@ static void acpi_sleep_run_lps0_dsm(unsigned int func, unsigned int func_mask, g rev_id, func, NULL); ACPI_FREE(out_obj); - acpi_handle_debug(lps0_device_handle, "_DSM function %u evaluation %s\n", - func, out_obj ? "successful" : "failed"); + lps0_dsm_state = func; + if (pm_debug_messages_on) { + acpi_handle_info(lps0_device_handle, + "%s transitioned to state %s\n", + out_obj ? "Successfully" : "Failed to", + acpi_sleep_dsm_state_to_str(lps0_dsm_state)); + } } -static bool acpi_s2idle_vendor_amd(void) -{ - return boot_cpu_data.x86_vendor == X86_VENDOR_AMD; -} static int validate_dsm(acpi_handle handle, const char *uuid, int rev, guid_t *dsm_guid) { From c9a236419ff936755eb5db8a894c3047440e65a8 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 2 Jun 2023 02:30:24 -0500 Subject: [PATCH 28/42] pinctrl: amd: Use pm_pr_dbg to show debugging messages To make the GPIO tracking around suspend easier for end users to use, link it with pm_debug_messages. This will make discovering sources of spurious GPIOs around suspend easier. Signed-off-by: Mario Limonciello Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- drivers/pinctrl/pinctrl-amd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c index f279b360c20d..43d3530bab48 100644 --- a/drivers/pinctrl/pinctrl-amd.c +++ b/drivers/pinctrl/pinctrl-amd.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "core.h" #include "pinctrl-utils.h" @@ -636,9 +637,8 @@ static bool do_amd_gpio_irq_handler(int irq, void *dev_id) regval = readl(regs + i); if (regval & PIN_IRQ_PENDING) - dev_dbg(&gpio_dev->pdev->dev, - "GPIO %d is active: 0x%x", - irqnr + i, regval); + pm_pr_dbg("GPIO %d is active: 0x%x", + irqnr + i, regval); /* caused wake on resume context for shared IRQ */ if (irq < 0 && (regval & BIT(WAKE_STS_OFF))) From b77505ed8a885c67a589c049c38824082a569068 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 2 Jun 2023 02:30:25 -0500 Subject: [PATCH 29/42] platform/x86/amd: pmc: Use pm_pr_dbg() for suspend related messages Using pm_pr_dbg() allows users to toggle `/sys/power/pm_debug_messages` as a single knob to turn on messages that amd-pmc can emit to aid in any s2idle debugging. Signed-off-by: Mario Limonciello Acked-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/platform/x86/amd/pmc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/platform/x86/amd/pmc.c b/drivers/platform/x86/amd/pmc.c index 427905714f79..1304cd6f13f6 100644 --- a/drivers/platform/x86/amd/pmc.c +++ b/drivers/platform/x86/amd/pmc.c @@ -543,7 +543,7 @@ static int amd_pmc_idlemask_read(struct amd_pmc_dev *pdev, struct device *dev, } if (dev) - dev_dbg(pdev->dev, "SMU idlemask s0i3: 0x%x\n", val); + pm_pr_dbg("SMU idlemask s0i3: 0x%x\n", val); if (s) seq_printf(s, "SMU idlemask : 0x%x\n", val); @@ -769,7 +769,7 @@ static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg) *arg |= (duration << 16); rc = rtc_alarm_irq_enable(rtc_device, 0); - dev_dbg(pdev->dev, "wakeup timer programmed for %lld seconds\n", duration); + pm_pr_dbg("wakeup timer programmed for %lld seconds\n", duration); return rc; } From 4622ba923e55e12cb76081c8865b01fcb383a9d8 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Mon, 5 Jun 2023 15:47:13 +0000 Subject: [PATCH 30/42] intel_idle: refactor state->enter manipulation into its own function Since the 6.4 kernel, the logic for updating a state's enter method based on "environmental conditions" (command line options, cpu sidechannel workarounds etc etc) has gotten pretty complex. This patch refactors this into a seperate small, self contained function (no behavior changes) for improved readability and to make future changes to this logic easier to do and understand. Signed-off-by: Arjan van de Ven Signed-off-by: Rafael J. Wysocki --- drivers/idle/intel_idle.c | 50 ++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index aa2d19db2b1d..c351b21c0875 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1839,6 +1839,32 @@ static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) return true; } +static void state_update_enter_method(struct cpuidle_state *state, int cstate) +{ + if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { + /* + * Combining with XSTATE with IBRS or IRQ_ENABLE flags + * is not currently supported but this driver. + */ + WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); + WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); + state->enter = intel_idle_xstate; + } else if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && + state->flags & CPUIDLE_FLAG_IBRS) { + /* + * IBRS mitigation requires that C-states are entered + * with interrupts disabled. + */ + WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); + state->enter = intel_idle_ibrs; + } else if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { + state->enter = intel_idle_irq; + } else if (force_irq_on) { + pr_info("forced intel_idle_irq for state %d\n", cstate); + state->enter = intel_idle_irq; + } +} + static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) { int cstate; @@ -1894,28 +1920,8 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) drv->states[drv->state_count] = cpuidle_state_table[cstate]; state = &drv->states[drv->state_count]; - if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { - /* - * Combining with XSTATE with IBRS or IRQ_ENABLE flags - * is not currently supported but this driver. - */ - WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); - WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); - state->enter = intel_idle_xstate; - } else if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && - state->flags & CPUIDLE_FLAG_IBRS) { - /* - * IBRS mitigation requires that C-states are entered - * with interrupts disabled. - */ - WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); - state->enter = intel_idle_ibrs; - } else if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { - state->enter = intel_idle_irq; - } else if (force_irq_on) { - pr_info("forced intel_idle_irq for state %d\n", cstate); - state->enter = intel_idle_irq; - } + state_update_enter_method(state, cstate); + if ((disabled_states_mask & BIT(drv->state_count)) || ((icpu->use_acpi || force_use_acpi) && From 7826c069c8765969cfb7a364f8e0e663fc132b10 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Mon, 5 Jun 2023 15:47:14 +0000 Subject: [PATCH 31/42] intel_idle: clean up the (new) state_update_enter_method function Now that the logic for state_update_enter_method() is in its own function, the long if .. else if .. else if .. else if chain can be simplified by just returning from the function at the various places. This does not change functionality, but it makes the logic much simpler to read or modify later. Signed-off-by: Arjan van de Ven Signed-off-by: Rafael J. Wysocki --- drivers/idle/intel_idle.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index c351b21c0875..256c2d42e350 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1849,7 +1849,10 @@ static void state_update_enter_method(struct cpuidle_state *state, int cstate) WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); state->enter = intel_idle_xstate; - } else if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && + return; + } + + if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && state->flags & CPUIDLE_FLAG_IBRS) { /* * IBRS mitigation requires that C-states are entered @@ -1857,9 +1860,15 @@ static void state_update_enter_method(struct cpuidle_state *state, int cstate) */ WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); state->enter = intel_idle_ibrs; - } else if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { + return; + } + + if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { state->enter = intel_idle_irq; - } else if (force_irq_on) { + return; + } + + if (force_irq_on) { pr_info("forced intel_idle_irq for state %d\n", cstate); state->enter = intel_idle_irq; } From b4a11fa3331e163e177e76098fe1d8b12b87cf6b Mon Sep 17 00:00:00 2001 From: Wyes Karny Date: Mon, 29 May 2023 14:25:51 +0000 Subject: [PATCH 32/42] cpufreq: Fail driver register if it has adjust_perf without fast_switch If fast_switch_possible flag is set by the scaling driver, the governor is free to select fast_switch function even if adjust_perf is set. Some scaling drivers which use adjust_perf don't set fast_switch thinking that the governor would never fall back to fast_switch. But the governor can fall back to fast_switch even in runtime if frequency invariance is disabled due to some reason. This could crash the kernel if the driver didn't set the fast_switch function pointer. Therefore, fail driver registration if it has adjust_perf without fast_switch. Suggested-by: Rafael J. Wysocki Suggested-by: Viresh Kumar Signed-off-by: Wyes Karny Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 3 ++- include/linux/cpufreq.h | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 6b52ebe5a890..50bbc969ffe5 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2828,7 +2828,8 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) (driver_data->setpolicy && (driver_data->target_index || driver_data->target)) || (!driver_data->get_intermediate != !driver_data->target_intermediate) || - (!driver_data->online != !driver_data->offline)) + (!driver_data->online != !driver_data->offline) || + (driver_data->adjust_perf && !driver_data->fast_switch)) return -EINVAL; pr_debug("trying to register driver %s\n", driver_data->name); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 26e2eb399484..172ff51c1b2a 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -340,7 +340,10 @@ struct cpufreq_driver { /* * ->fast_switch() replacement for drivers that use an internal * representation of performance levels and can pass hints other than - * the target performance level to the hardware. + * the target performance level to the hardware. This can only be set + * if ->fast_switch is set too, because in those cases (under specific + * conditions) scale invariance can be disabled, which causes the + * schedutil governor to fall back to the latter. */ void (*adjust_perf)(unsigned int cpu, unsigned long min_perf, From 2f3d08f074b02aa449de27238fda72496c789034 Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Mon, 5 Jun 2023 15:47:15 +0000 Subject: [PATCH 33/42] intel_idle: Add support for using intel_idle in a VM guest using just hlt In a typical VM guest, the mwait instruction is not available, leaving only the 'hlt' instruction (which causes a VMEXIT to the host). So for this common case, intel_idle will detect the lack of mwait, and fail to initialize (after which another idle method would step in which will just use hlt always). Other (non-common) cases exist; the table below shows the before/after for these: +------------+--------------------------+-------------------------+ | Hypervisor | Idle method before patch | Idle method after patch | | exposes | | | +============+==========================+=========================+ | nothing | default_idle fallback | intel_idle VM table | | (common) | (straight "hlt") | | +------------+--------------------------+-------------------------+ | mwait | intel_idle mwait table | intel_idle mwait table | +------------+--------------------------+-------------------------+ | ACPI | ACPI C1 state ("hlt") | intel_idle VM table | +------------+--------------------------+-------------------------+ This is only applicable to CPUs known by intel_idle. For the bare metal case, unknown CPU models will use the ACPI tables (when available) to get estimates for latency and break even point for longer idle states. In guests, the common case is that ACPI tables are not available, but even when they are available, they can't and don't provide the latency information for the longer (mwait based) states. For this scenario (unknown CPU model), the default_idle mode (no ACPI) or ACPI C1 (ACPI avaible) will be used. By providing capability to do this with the intel_idle driver, we can do better than the fallback or ACPI table methods. While this current change only gets us to the existing behavior, later patches in this series will add new capabilities such as optimized TLB flushing. In order to do this, a simplified version of the initialization function for VM guests is created, and this will be called if the CPU is recognized, but mwait is not supported, and we're in a VM guest. One thing to note is that the max latency (and break even) of this C1 state is higher than the typical bare metal C1 state. Because hlt causes a vmexit, and the cost of vmexit + hypervisor overhead + vmenter is typically in the order of upto 5 microseconds... even if the hypervisor does not actually goes into a hardware power saving state. Signed-off-by: Arjan van de Ven [ rjw: Dropped redundant checks from should_verify_mwait() ] Signed-off-by: Rafael J. Wysocki --- drivers/idle/intel_idle.c | 117 +++++++++++++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 1 deletion(-) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 256c2d42e350..a80e1f520293 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -199,6 +199,43 @@ static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, return __intel_idle(dev, drv, index); } +static __always_inline int __intel_idle_hlt(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) +{ + raw_safe_halt(); + raw_local_irq_disable(); + return index; +} + +/** + * intel_idle_hlt - Ask the processor to enter the given idle state using hlt. + * @dev: cpuidle device of the target CPU. + * @drv: cpuidle driver (assumed to point to intel_idle_driver). + * @index: Target idle state index. + * + * Use the HLT instruction to notify the processor that the CPU represented by + * @dev is idle and it can try to enter the idle state corresponding to @index. + * + * Must be called under local_irq_disable(). + */ +static __cpuidle int intel_idle_hlt(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) +{ + return __intel_idle_hlt(dev, drv, index); +} + +static __cpuidle int intel_idle_hlt_irq_on(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) +{ + int ret; + + raw_local_irq_enable(); + ret = __intel_idle_hlt(dev, drv, index); + raw_local_irq_disable(); + + return ret; +} + /** * intel_idle_s2idle - Ask the processor to enter the given idle state. * @dev: cpuidle device of the target CPU. @@ -1242,6 +1279,18 @@ static struct cpuidle_state snr_cstates[] __initdata = { .enter = NULL } }; +static struct cpuidle_state vmguest_cstates[] __initdata = { + { + .name = "C1", + .desc = "HLT", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, + .exit_latency = 5, + .target_residency = 10, + .enter = &intel_idle_hlt, }, + { + .enter = NULL } +}; + static const struct idle_cpu idle_cpu_nehalem __initconst = { .state_table = nehalem_cstates, .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, @@ -1841,6 +1890,16 @@ static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) static void state_update_enter_method(struct cpuidle_state *state, int cstate) { + if (state->enter == intel_idle_hlt) { + if (force_irq_on) { + pr_info("forced intel_idle_irq for state %d\n", cstate); + state->enter = intel_idle_hlt_irq_on; + } + return; + } + if (state->enter == intel_idle_hlt_irq_on) + return; /* no update scenarios */ + if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { /* * Combining with XSTATE with IBRS or IRQ_ENABLE flags @@ -1874,6 +1933,21 @@ static void state_update_enter_method(struct cpuidle_state *state, int cstate) } } +/* + * For mwait based states, we want to verify the cpuid data to see if the state + * is actually supported by this specific CPU. + * For non-mwait based states, this check should be skipped. + */ +static bool should_verify_mwait(struct cpuidle_state *state) +{ + if (state->enter == intel_idle_hlt) + return false; + if (state->enter == intel_idle_hlt_irq_on) + return false; + + return true; +} + static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) { int cstate; @@ -1922,7 +1996,7 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) } mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); - if (!intel_idle_verify_cstate(mwait_hint)) + if (should_verify_mwait(&cpuidle_state_table[cstate]) && !intel_idle_verify_cstate(mwait_hint)) continue; /* Structure copy. */ @@ -2056,6 +2130,45 @@ static void __init intel_idle_cpuidle_devices_uninit(void) cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); } +static int __init intel_idle_vminit(const struct x86_cpu_id *id) +{ + int retval; + + cpuidle_state_table = vmguest_cstates; + + icpu = (const struct idle_cpu *)id->driver_data; + + pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", + boot_cpu_data.x86_model); + + intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); + if (!intel_idle_cpuidle_devices) + return -ENOMEM; + + intel_idle_cpuidle_driver_init(&intel_idle_driver); + + retval = cpuidle_register_driver(&intel_idle_driver); + if (retval) { + struct cpuidle_driver *drv = cpuidle_get_driver(); + printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), + drv ? drv->name : "none"); + goto init_driver_fail; + } + + retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", + intel_idle_cpu_online, NULL); + if (retval < 0) + goto hp_setup_fail; + + return 0; +hp_setup_fail: + intel_idle_cpuidle_devices_uninit(); + cpuidle_unregister_driver(&intel_idle_driver); +init_driver_fail: + free_percpu(intel_idle_cpuidle_devices); + return retval; +} + static int __init intel_idle_init(void) { const struct x86_cpu_id *id; @@ -2074,6 +2187,8 @@ static int __init intel_idle_init(void) id = x86_match_cpu(intel_idle_ids); if (id) { if (!boot_cpu_has(X86_FEATURE_MWAIT)) { + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + return intel_idle_vminit(id); pr_debug("Please enable MWAIT in BIOS SETUP\n"); return -ENODEV; } From 217e67784eab30cd0704fab4109647ea68a4d850 Mon Sep 17 00:00:00 2001 From: Wyes Karny Date: Tue, 30 May 2023 13:13:48 +0000 Subject: [PATCH 34/42] cpufreq: amd-pstate: Write CPPC enable bit per-socket Currently amd_pstate sets CPPC enable bit in MSR_AMD_CPPC_ENABLE only for the CPU where the module_init happened. But MSR_AMD_CPPC_ENABLE is per-socket. This causes CPPC enable bit to set for only one socket for servers with more than one physical packages. To fix this write MSR_AMD_CPPC_ENABLE per-socket. Also, handle duplicate calls for cppc_enable, because it's called from per-policy/per-core callbacks and can result in duplicate MSR writes. Before the fix: amd@amd:~$ sudo rdmsr -a 0xc00102b1 | uniq --count 192 0 192 1 After the fix: amd@amd:~$ sudo rdmsr -a 0xc00102b1 | uniq --count 384 1 Suggested-by: Gautham R. Shenoy Signed-off-by: Wyes Karny Acked-by: Huang Rui Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ddd346a239e0..50722bfbb34a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -63,6 +63,7 @@ static struct cpufreq_driver *current_pstate_driver; static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_DISABLE; +static bool cppc_enabled; /* * AMD Energy Preference Performance (EPP) @@ -228,7 +229,28 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, static inline int pstate_enable(bool enable) { - return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); + int ret, cpu; + unsigned long logical_proc_id_mask = 0; + + if (enable == cppc_enabled) + return 0; + + for_each_present_cpu(cpu) { + unsigned long logical_id = topology_logical_die_id(cpu); + + if (test_bit(logical_id, &logical_proc_id_mask)) + continue; + + set_bit(logical_id, &logical_proc_id_mask); + + ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE, + enable); + if (ret) + return ret; + } + + cppc_enabled = enable; + return 0; } static int cppc_enable(bool enable) @@ -236,6 +258,9 @@ static int cppc_enable(bool enable) int cpu, ret = 0; struct cppc_perf_ctrls perf_ctrls; + if (enable == cppc_enabled) + return 0; + for_each_present_cpu(cpu) { ret = cppc_set_enable(cpu, enable); if (ret) @@ -251,6 +276,7 @@ static int cppc_enable(bool enable) } } + cppc_enabled = enable; return ret; } From f4aad639302a07454dcb23b408dcadf8a9efb031 Mon Sep 17 00:00:00 2001 From: Wyes Karny Date: Mon, 12 Jun 2023 11:36:10 +0000 Subject: [PATCH 35/42] cpufreq: amd-pstate: Make amd-pstate EPP driver name hyphenated amd-pstate passive mode driver is hyphenated. So make amd-pstate active mode driver consistent with that rename "amd_pstate_epp" to "amd-pstate-epp". Fixes: ffa5096a7c33 ("cpufreq: amd-pstate: implement Pstate EPP support for the AMD processors") Cc: All applicable Reviewed-by: Gautham R. Shenoy Signed-off-by: Wyes Karny Acked-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 50722bfbb34a..d8269994322e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1382,7 +1382,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .online = amd_pstate_epp_cpu_online, .suspend = amd_pstate_epp_suspend, .resume = amd_pstate_epp_resume, - .name = "amd_pstate_epp", + .name = "amd-pstate-epp", .attr = amd_pstate_epp_attr, }; From 4384a70c8813e8573d1841fd94eee873f80a7e1a Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 30 May 2023 11:55:36 +0200 Subject: [PATCH 36/42] PM: domains: Move the verification of in-params from genpd_add_device() Commit f38d1a6d0025 ("PM: domains: Allocate governor data dynamically based on a genpd governor") started to use the in-parameters in genpd_add_device(), without first doing a verification of them. This isn't really a big problem, as most callers do a verification already. Therefore, let's drop the verification from genpd_add_device() and make sure all the callers take care of it instead. Reported-by: Dan Carpenter Fixes: f38d1a6d0025 ("PM: domains: Allocate governor data dynamically based on a genpd governor") Signed-off-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 51b9d4eaab5e..5cb2023581d4 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -1632,9 +1632,6 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, dev_dbg(dev, "%s()\n", __func__); - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev)) - return -EINVAL; - gpd_data = genpd_alloc_dev_data(dev, gd); if (IS_ERR(gpd_data)) return PTR_ERR(gpd_data); @@ -1676,6 +1673,9 @@ int pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev) { int ret; + if (!genpd || !dev) + return -EINVAL; + mutex_lock(&gpd_list_lock); ret = genpd_add_device(genpd, dev, dev); mutex_unlock(&gpd_list_lock); @@ -2523,6 +2523,9 @@ int of_genpd_add_device(struct of_phandle_args *genpdspec, struct device *dev) struct generic_pm_domain *genpd; int ret; + if (!dev) + return -EINVAL; + mutex_lock(&gpd_list_lock); genpd = genpd_get_from_provider(genpdspec); From a4ba10bf6855bf9381fe2365ec9c3af84c1fa7db Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 12 Jun 2023 09:26:48 -0400 Subject: [PATCH 37/42] cpufreq: amd-pstate: Set default governor to schedutil The Kconfig currently defaults the governor to schedutil on x86_64 only when intel-pstate and SMP have been selected. If the kernel is built only with amd-pstate, the default governor should also be schedutil. Signed-off-by: Mario Limonciello Reviewed-by: Leo Li Acked-by: Huang Rui Tested-by: Perry Yuan Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 2c839bd2b051..a1c51abddbc5 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -38,7 +38,7 @@ choice prompt "Default CPUFreq governor" default CPU_FREQ_DEFAULT_GOV_USERSPACE if ARM_SA1110_CPUFREQ default CPU_FREQ_DEFAULT_GOV_SCHEDUTIL if ARM64 || ARM - default CPU_FREQ_DEFAULT_GOV_SCHEDUTIL if X86_INTEL_PSTATE && SMP + default CPU_FREQ_DEFAULT_GOV_SCHEDUTIL if (X86_INTEL_PSTATE || X86_AMD_PSTATE) && SMP default CPU_FREQ_DEFAULT_GOV_PERFORMANCE help This option sets which CPUFreq governor shall be loaded at From 965262ef71c475857d0984a5eee57694b207a113 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 20 Jun 2023 12:24:31 -0500 Subject: [PATCH 38/42] ACPI: CPPC: Add definition for undefined FADT preferred PM profile value In the event a new preferred PM profile value is introduced it's best for code to be able to defensively guard against it so that the wrong settings don't get applied on a new system that uses this profile but ancient kernels. Acked-by: Huang Rui Suggested-by: Gautham Ranjal Shenoy Link: https://uefi.org/htmlspecs/ACPI_Spec_6_4_html/05_ACPI_Software_Programming_Model/ACPI_Software_Programming_Model.html#fixed-acpi-description-table-fadt Signed-off-by: Mario Limonciello Reviewed-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- include/acpi/actbl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/acpi/actbl.h b/include/acpi/actbl.h index e5dfb6f4de52..451f6276da49 100644 --- a/include/acpi/actbl.h +++ b/include/acpi/actbl.h @@ -307,7 +307,8 @@ enum acpi_preferred_pm_profiles { PM_SOHO_SERVER = 5, PM_APPLIANCE_PC = 6, PM_PERFORMANCE_SERVER = 7, - PM_TABLET = 8 + PM_TABLET = 8, + NR_PM_PROFILES = 9 }; /* Values for sleep_status and sleep_control registers (V5+ FADT) */ From 32f80b9adfdb43f8af248596724f59dde938a190 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 20 Jun 2023 12:24:32 -0500 Subject: [PATCH 39/42] cpufreq: amd-pstate: Set a fallback policy based on preferred_profile If a user's configuration doesn't explicitly specify the cpufreq scaling governor then the code currently explicitly falls back to 'powersave'. This default is fine for notebooks and desktops, but servers and undefined machines should default to 'performance'. Look at the 'preferred_profile' field from the FADT to set this policy accordingly. Link: https://uefi.org/htmlspecs/ACPI_Spec_6_4_html/05_ACPI_Software_Programming_Model/ACPI_Software_Programming_Model.html#fixed-acpi-description-table-fadt Acked-by: Huang Rui Suggested-by: Wyes Karny Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Reviewed-by: Perry Yuan Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d8269994322e..3546d7db614d 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1071,6 +1071,26 @@ static const struct attribute_group amd_pstate_global_attr_group = { .attrs = pstate_global_attributes, }; +static bool amd_pstate_acpi_pm_profile_server(void) +{ + switch (acpi_gbl_FADT.preferred_profile) { + case PM_ENTERPRISE_SERVER: + case PM_SOHO_SERVER: + case PM_PERFORMANCE_SERVER: + return true; + } + return false; +} + +static bool amd_pstate_acpi_pm_profile_undefined(void) +{ + if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) + return true; + if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) + return true; + return false; +} + static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -1128,10 +1148,14 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->max = policy->cpuinfo.max_freq; /* - * Set the policy to powersave to provide a valid fallback value in case + * Set the policy to provide a valid fallback value in case * the default cpufreq governor is neither powersave nor performance. */ - policy->policy = CPUFREQ_POLICY_POWERSAVE; + if (amd_pstate_acpi_pm_profile_server() || + amd_pstate_acpi_pm_profile_undefined()) + policy->policy = CPUFREQ_POLICY_PERFORMANCE; + else + policy->policy = CPUFREQ_POLICY_POWERSAVE; if (boot_cpu_has(X86_FEATURE_CPPC)) { ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); From c88ad30e3f861c7be4e3b4995554e2b0754059b7 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 20 Jun 2023 12:24:33 -0500 Subject: [PATCH 40/42] cpufreq: amd-pstate: Add a kernel config option to set default mode Users are having more success with amd-pstate since the introduction of EPP and Guided modes. To expose the driver to more users by default introduce a kernel configuration option for setting the default mode. Users can use an integer to map out which default mode they want to use in lieu of a kernel command line option. This will default to EPP, but only if: 1) The CPU supports an MSR. 2) The system profile is identified 3) The system profile is identified as a non-server by the FADT. Link: https://gitlab.freedesktop.org/hadess/power-profiles-daemon/-/merge_requests/121 Acked-by: Huang Rui Reviewed-by: Gautham R. Shenoy Co-developed-by: Perry Yuan Signed-off-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/Kconfig.x86 | 17 +++++++++ drivers/cpufreq/amd-pstate.c | 73 ++++++++++++++++++++++++------------ include/linux/amd-pstate.h | 4 +- 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 00476e94db90..438c9e75a04d 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -51,6 +51,23 @@ config X86_AMD_PSTATE If in doubt, say N. +config X86_AMD_PSTATE_DEFAULT_MODE + int "AMD Processor P-State default mode" + depends on X86_AMD_PSTATE + default 3 if X86_AMD_PSTATE + range 1 4 + help + Select the default mode the amd-pstate driver will use on + supported hardware. + The value set has the following meanings: + 1 -> Disabled + 2 -> Passive + 3 -> Active (EPP) + 4 -> Guided + + For details, take a look at: + . + config X86_AMD_PSTATE_UT tristate "selftest for AMD Processor P-State driver" depends on X86 && ACPI_PROCESSOR diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 3546d7db614d..81fba0dcbee9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -62,7 +62,7 @@ static struct cpufreq_driver *current_pstate_driver; static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; -static int cppc_state = AMD_PSTATE_DISABLE; +static int cppc_state = AMD_PSTATE_UNDEFINED; static bool cppc_enabled; /* @@ -1410,6 +1410,25 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .attr = amd_pstate_epp_attr, }; +static int __init amd_pstate_set_driver(int mode_idx) +{ + if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { + cppc_state = mode_idx; + if (cppc_state == AMD_PSTATE_DISABLE) + pr_info("driver is explicitly disabled\n"); + + if (cppc_state == AMD_PSTATE_ACTIVE) + current_pstate_driver = &amd_pstate_epp_driver; + + if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) + current_pstate_driver = &amd_pstate_driver; + + return 0; + } + + return -EINVAL; +} + static int __init amd_pstate_init(void) { struct device *dev_root; @@ -1417,15 +1436,6 @@ static int __init amd_pstate_init(void) if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) return -ENODEV; - /* - * by default the pstate driver is disabled to load - * enable the amd_pstate passive mode driver explicitly - * with amd_pstate=passive or other modes in kernel command line - */ - if (cppc_state == AMD_PSTATE_DISABLE) { - pr_info("driver load is disabled, boot with specific mode to enable this\n"); - return -ENODEV; - } if (!acpi_cpc_valid()) { pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); @@ -1436,6 +1446,33 @@ static int __init amd_pstate_init(void) if (cpufreq_get_current_driver()) return -EEXIST; + switch (cppc_state) { + case AMD_PSTATE_UNDEFINED: + /* Disable on the following configs by default: + * 1. Undefined platforms + * 2. Server platforms + * 3. Shared memory designs + */ + if (amd_pstate_acpi_pm_profile_undefined() || + amd_pstate_acpi_pm_profile_server() || + !boot_cpu_has(X86_FEATURE_CPPC)) { + pr_info("driver load is disabled, boot with specific mode to enable this\n"); + return -ENODEV; + } + ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE); + if (ret) + return ret; + break; + case AMD_PSTATE_DISABLE: + return -ENODEV; + case AMD_PSTATE_PASSIVE: + case AMD_PSTATE_ACTIVE: + case AMD_PSTATE_GUIDED: + break; + default: + return -EINVAL; + } + /* capability check */ if (boot_cpu_has(X86_FEATURE_CPPC)) { pr_debug("AMD CPPC MSR based functionality is supported\n"); @@ -1488,21 +1525,7 @@ static int __init amd_pstate_param(char *str) size = strlen(str); mode_idx = get_mode_idx_from_str(str, size); - if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { - cppc_state = mode_idx; - if (cppc_state == AMD_PSTATE_DISABLE) - pr_info("driver is explicitly disabled\n"); - - if (cppc_state == AMD_PSTATE_ACTIVE) - current_pstate_driver = &amd_pstate_epp_driver; - - if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) - current_pstate_driver = &amd_pstate_driver; - - return 0; - } - - return -EINVAL; + return amd_pstate_set_driver(mode_idx); } early_param("amd_pstate", amd_pstate_param); diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index c10ebf8c42e6..446394f84606 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -94,7 +94,8 @@ struct amd_cpudata { * enum amd_pstate_mode - driver working mode of amd pstate */ enum amd_pstate_mode { - AMD_PSTATE_DISABLE = 0, + AMD_PSTATE_UNDEFINED = 0, + AMD_PSTATE_DISABLE, AMD_PSTATE_PASSIVE, AMD_PSTATE_ACTIVE, AMD_PSTATE_GUIDED, @@ -102,6 +103,7 @@ enum amd_pstate_mode { }; static const char * const amd_pstate_mode_string[] = { + [AMD_PSTATE_UNDEFINED] = "undefined", [AMD_PSTATE_DISABLE] = "disable", [AMD_PSTATE_PASSIVE] = "passive", [AMD_PSTATE_ACTIVE] = "active", From 03f44ffb3d5be2fceda375d92c70ab6de4df7081 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 21 Jun 2023 09:58:39 +0300 Subject: [PATCH 41/42] cpufreq: intel_pstate: Fix energy_performance_preference for passive If the intel_pstate driver is set to passive mode, then writing the same value to the energy_performance_preference sysfs twice will fail. This is caused by the wrong return value used (index of the matched energy_perf_string), instead of the length of the passed in parameter. Fix by forcing the internal return value to zero when the same preference is passed in by user. This same issue is not present when active mode is used for the driver. Fixes: f6ebbcf08f37 ("cpufreq: intel_pstate: Implement passive mode with HWP enabled") Reported-by: Niklas Neronin Signed-off-by: Tero Kristo Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 2548ec92faa2..f29182512b98 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -824,6 +824,8 @@ static ssize_t store_energy_performance_preference( err = cpufreq_start_governor(policy); if (!ret) ret = err; + } else { + ret = 0; } } From 0fac214bb75ee64d7b9e6521d53f8251a4d324ea Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Tue, 20 Jun 2023 18:03:54 +0000 Subject: [PATCH 42/42] intel_idle: Add a "Long HLT" C1 state for the VM guest mode intel_idle will, for the bare metal case, usually have one or more deep power states that have the CPUIDLE_FLAG_TLB_FLUSHED flag set. When a state with this flag is selected by the cpuidle framework, it will also flush the TLBs as part of entering this state. The benefit of doing this is that the kernel does not need to wake the cpu out of this deep power state just to flush the TLBs... for which the latency can be very high due to the exit latency of deep power states. In a VM guest currently, this benefit of avoiding the wakeup does not exist, while the problem (long exit latency) is even more severe. Linux will need to wake up a vCPU (causing the host to either come out of a deep C state, or the VMM to have to deschedule something else to schedule the vCPU) which can take a very long time.. adding a lot of latency to tlb flush operations (including munmap and others). To solve this, add a "Long HLT" C state to the state table for the VM guest case that has the CPUIDLE_FLAG_TLB_FLUSHED flag set. The result of that is that for long idle periods (where the VMM is likely to do things that cause large latency) the cpuidle framework will flush the TLBs (and avoid the wakeups), while for short/quick idle durations, the existing behavior is retained. Now, there is still only "hlt" available in the guest, but for long idle, the host can go to a deeper state (say C6). There is a reasonable debate one can have to what to set for the exit_latency and break even point for this "Long HLT" state. The good news is that intel_idle has these values available for the underlying CPU (even when mwait is not exposed). The solution thus is to just use the latency and break even of the deepest state from the bare metal CPU. This is under the assumption that this is a pretty reasonable estimate of what the VMM would do to cause latency. Signed-off-by: Arjan van de Ven Signed-off-by: Rafael J. Wysocki --- drivers/idle/intel_idle.c | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index a80e1f520293..34201d7ef33e 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1287,6 +1287,13 @@ static struct cpuidle_state vmguest_cstates[] __initdata = { .exit_latency = 5, .target_residency = 10, .enter = &intel_idle_hlt, }, + { + .name = "C1L", + .desc = "Long HLT", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 5, + .target_residency = 200, + .enter = &intel_idle_hlt, }, { .enter = NULL } }; @@ -2130,6 +2137,45 @@ static void __init intel_idle_cpuidle_devices_uninit(void) cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); } +/* + * Match up the latency and break even point of the bare metal (cpu based) + * states with the deepest VM available state. + * + * We only want to do this for the deepest state, the ones that has + * the TLB_FLUSHED flag set on the . + * + * All our short idle states are dominated by vmexit/vmenter latencies, + * not the underlying hardware latencies so we keep our values for these. + */ +static void matchup_vm_state_with_baremetal(void) +{ + int cstate; + + for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { + int matching_cstate; + + if (intel_idle_max_cstate_reached(cstate)) + break; + + if (!cpuidle_state_table[cstate].enter) + break; + + if (!(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_TLB_FLUSHED)) + continue; + + for (matching_cstate = 0; matching_cstate < CPUIDLE_STATE_MAX; ++matching_cstate) { + if (!icpu->state_table[matching_cstate].enter) + break; + if (icpu->state_table[matching_cstate].exit_latency > cpuidle_state_table[cstate].exit_latency) { + cpuidle_state_table[cstate].exit_latency = icpu->state_table[matching_cstate].exit_latency; + cpuidle_state_table[cstate].target_residency = icpu->state_table[matching_cstate].target_residency; + } + } + + } +} + + static int __init intel_idle_vminit(const struct x86_cpu_id *id) { int retval; @@ -2145,6 +2191,15 @@ static int __init intel_idle_vminit(const struct x86_cpu_id *id) if (!intel_idle_cpuidle_devices) return -ENOMEM; + /* + * We don't know exactly what the host will do when we go idle, but as a worst estimate + * we can assume that the exit latency of the deepest host state will be hit for our + * deep (long duration) guest idle state. + * The same logic applies to the break even point for the long duration guest idle state. + * So lets copy these two properties from the table we found for the host CPU type. + */ + matchup_vm_state_with_baremetal(); + intel_idle_cpuidle_driver_init(&intel_idle_driver); retval = cpuidle_register_driver(&intel_idle_driver);