summaryrefslogtreecommitdiffstats
path: root/drivers/cpufreq
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
commitace9429bb58fd418f0c81d4c2835699bddf6bde6 (patch)
treeb2d64bc10158fdd5497876388cd68142ca374ed3 /drivers/cpufreq
parentInitial commit. (diff)
downloadlinux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.tar.xz
linux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.zip
Adding upstream version 6.6.15.upstream/6.6.15
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'drivers/cpufreq')
-rw-r--r--drivers/cpufreq/Kconfig316
-rw-r--r--drivers/cpufreq/Kconfig.arm287
-rw-r--r--drivers/cpufreq/Kconfig.powerpc56
-rw-r--r--drivers/cpufreq/Kconfig.x86341
-rw-r--r--drivers/cpufreq/Makefile109
-rw-r--r--drivers/cpufreq/acpi-cpufreq.c1048
-rw-r--r--drivers/cpufreq/amd-pstate-trace.c2
-rw-r--r--drivers/cpufreq/amd-pstate-trace.h97
-rw-r--r--drivers/cpufreq/amd-pstate-ut.c283
-rw-r--r--drivers/cpufreq/amd-pstate.c1573
-rw-r--r--drivers/cpufreq/amd_freq_sensitivity.c159
-rw-r--r--drivers/cpufreq/apple-soc-cpufreq.c353
-rw-r--r--drivers/cpufreq/armada-37xx-cpufreq.c562
-rw-r--r--drivers/cpufreq/armada-8k-cpufreq.c217
-rw-r--r--drivers/cpufreq/bmips-cpufreq.c189
-rw-r--r--drivers/cpufreq/brcmstb-avs-cpufreq.c781
-rw-r--r--drivers/cpufreq/cppc_cpufreq.c1017
-rw-r--r--drivers/cpufreq/cpufreq-dt-platdev.c228
-rw-r--r--drivers/cpufreq/cpufreq-dt.c371
-rw-r--r--drivers/cpufreq/cpufreq-dt.h25
-rw-r--r--drivers/cpufreq/cpufreq-nforce2.c438
-rw-r--r--drivers/cpufreq/cpufreq.c3005
-rw-r--r--drivers/cpufreq/cpufreq_conservative.c344
-rw-r--r--drivers/cpufreq/cpufreq_governor.c581
-rw-r--r--drivers/cpufreq/cpufreq_governor.h181
-rw-r--r--drivers/cpufreq/cpufreq_governor_attr_set.c76
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.c486
-rw-r--r--drivers/cpufreq/cpufreq_ondemand.h26
-rw-r--r--drivers/cpufreq/cpufreq_performance.c45
-rw-r--r--drivers/cpufreq/cpufreq_powersave.c39
-rw-r--r--drivers/cpufreq/cpufreq_stats.c289
-rw-r--r--drivers/cpufreq/cpufreq_userspace.c142
-rw-r--r--drivers/cpufreq/davinci-cpufreq.c156
-rw-r--r--drivers/cpufreq/e_powersaver.c424
-rw-r--r--drivers/cpufreq/elanfreq.c228
-rw-r--r--drivers/cpufreq/freq_table.c374
-rw-r--r--drivers/cpufreq/gx-suspmod.c498
-rw-r--r--drivers/cpufreq/highbank-cpufreq.c113
-rw-r--r--drivers/cpufreq/ia64-acpi-cpufreq.c353
-rw-r--r--drivers/cpufreq/imx-cpufreq-dt.c195
-rw-r--r--drivers/cpufreq/imx6q-cpufreq.c547
-rw-r--r--drivers/cpufreq/intel_pstate.c3571
-rw-r--r--drivers/cpufreq/kirkwood-cpufreq.c203
-rw-r--r--drivers/cpufreq/longhaul.c1000
-rw-r--r--drivers/cpufreq/longhaul.h352
-rw-r--r--drivers/cpufreq/longrun.c317
-rw-r--r--drivers/cpufreq/loongson2_cpufreq.c184
-rw-r--r--drivers/cpufreq/maple-cpufreq.c241
-rw-r--r--drivers/cpufreq/mediatek-cpufreq-hw.c342
-rw-r--r--drivers/cpufreq/mediatek-cpufreq.c805
-rw-r--r--drivers/cpufreq/mvebu-cpufreq.c101
-rw-r--r--drivers/cpufreq/omap-cpufreq.c200
-rw-r--r--drivers/cpufreq/p4-clockmod.c273
-rw-r--r--drivers/cpufreq/pasemi-cpufreq.c275
-rw-r--r--drivers/cpufreq/pcc-cpufreq.c645
-rw-r--r--drivers/cpufreq/pmac32-cpufreq.c693
-rw-r--r--drivers/cpufreq/pmac64-cpufreq.c674
-rw-r--r--drivers/cpufreq/powernow-k6.c311
-rw-r--r--drivers/cpufreq/powernow-k7.c696
-rw-r--r--drivers/cpufreq/powernow-k7.h41
-rw-r--r--drivers/cpufreq/powernow-k8.c1221
-rw-r--r--drivers/cpufreq/powernow-k8.h188
-rw-r--r--drivers/cpufreq/powernv-cpufreq.c1166
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq.c173
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq.h33
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq_pervasive.c102
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq_pmi.c150
-rw-r--r--drivers/cpufreq/pxa2xx-cpufreq.c321
-rw-r--r--drivers/cpufreq/pxa3xx-cpufreq.c235
-rw-r--r--drivers/cpufreq/qcom-cpufreq-hw.c760
-rw-r--r--drivers/cpufreq/qcom-cpufreq-nvmem.c413
-rw-r--r--drivers/cpufreq/qoriq-cpufreq.c308
-rw-r--r--drivers/cpufreq/raspberrypi-cpufreq.c95
-rw-r--r--drivers/cpufreq/s3c64xx-cpufreq.c208
-rw-r--r--drivers/cpufreq/s5pv210-cpufreq.c687
-rw-r--r--drivers/cpufreq/sa1110-cpufreq.c367
-rw-r--r--drivers/cpufreq/sc520_freq.c138
-rw-r--r--drivers/cpufreq/scmi-cpufreq.c350
-rw-r--r--drivers/cpufreq/scpi-cpufreq.c229
-rw-r--r--drivers/cpufreq/sh-cpufreq.c175
-rw-r--r--drivers/cpufreq/sparc-us2e-cpufreq.c353
-rw-r--r--drivers/cpufreq/sparc-us3-cpufreq.c201
-rw-r--r--drivers/cpufreq/spear-cpufreq.c247
-rw-r--r--drivers/cpufreq/speedstep-centrino.c561
-rw-r--r--drivers/cpufreq/speedstep-ich.c386
-rw-r--r--drivers/cpufreq/speedstep-lib.c479
-rw-r--r--drivers/cpufreq/speedstep-lib.h48
-rw-r--r--drivers/cpufreq/speedstep-smi.c393
-rw-r--r--drivers/cpufreq/sti-cpufreq.c303
-rw-r--r--drivers/cpufreq/sun50i-cpufreq-nvmem.c217
-rw-r--r--drivers/cpufreq/tegra124-cpufreq.c222
-rw-r--r--drivers/cpufreq/tegra186-cpufreq.c285
-rw-r--r--drivers/cpufreq/tegra194-cpufreq.c769
-rw-r--r--drivers/cpufreq/tegra20-cpufreq.c113
-rw-r--r--drivers/cpufreq/ti-cpufreq.c462
-rw-r--r--drivers/cpufreq/vexpress-spc-cpufreq.c577
96 files changed, 40413 insertions, 0 deletions
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
new file mode 100644
index 0000000000..f429b9b37b
--- /dev/null
+++ b/drivers/cpufreq/Kconfig
@@ -0,0 +1,316 @@
+# SPDX-License-Identifier: GPL-2.0-only
+menu "CPU Frequency scaling"
+
+config CPU_FREQ
+ bool "CPU Frequency scaling"
+ help
+ CPU Frequency scaling allows you to change the clock speed of
+ CPUs on the fly. This is a nice method to save power, because
+ the lower the CPU clock speed, the less power the CPU consumes.
+
+ Note that this driver doesn't automatically change the CPU
+ clock speed, you need to either enable a dynamic cpufreq governor
+ (see below) after boot, or use a userspace tool.
+
+ For details, take a look at
+ <file:Documentation/admin-guide/pm/cpufreq.rst>.
+
+ If in doubt, say N.
+
+if CPU_FREQ
+
+config CPU_FREQ_GOV_ATTR_SET
+ bool
+
+config CPU_FREQ_GOV_COMMON
+ select CPU_FREQ_GOV_ATTR_SET
+ select IRQ_WORK
+ bool
+
+config CPU_FREQ_STAT
+ bool "CPU frequency transition statistics"
+ help
+ Export CPU frequency statistics information through sysfs.
+
+ If in doubt, say N.
+
+choice
+ prompt "Default CPUFreq governor"
+ default CPU_FREQ_DEFAULT_GOV_USERSPACE if ARM_SA1110_CPUFREQ
+ default CPU_FREQ_DEFAULT_GOV_SCHEDUTIL if ARM64 || ARM
+ default CPU_FREQ_DEFAULT_GOV_SCHEDUTIL if (X86_INTEL_PSTATE || X86_AMD_PSTATE) && SMP
+ default CPU_FREQ_DEFAULT_GOV_PERFORMANCE
+ help
+ This option sets which CPUFreq governor shall be loaded at
+ startup. If in doubt, use the default setting.
+
+config CPU_FREQ_DEFAULT_GOV_PERFORMANCE
+ bool "performance"
+ select CPU_FREQ_GOV_PERFORMANCE
+ help
+ Use the CPUFreq governor 'performance' as default. This sets
+ the frequency statically to the highest frequency supported by
+ the CPU.
+
+config CPU_FREQ_DEFAULT_GOV_POWERSAVE
+ bool "powersave"
+ select CPU_FREQ_GOV_POWERSAVE
+ help
+ Use the CPUFreq governor 'powersave' as default. This sets
+ the frequency statically to the lowest frequency supported by
+ the CPU.
+
+config CPU_FREQ_DEFAULT_GOV_USERSPACE
+ bool "userspace"
+ select CPU_FREQ_GOV_USERSPACE
+ help
+ Use the CPUFreq governor 'userspace' as default. This allows
+ you to set the CPU frequency manually or when a userspace
+ program shall be able to set the CPU dynamically without having
+ to enable the userspace governor manually.
+
+config CPU_FREQ_DEFAULT_GOV_ONDEMAND
+ bool "ondemand"
+ depends on !(X86_INTEL_PSTATE && SMP)
+ select CPU_FREQ_GOV_ONDEMAND
+ select CPU_FREQ_GOV_PERFORMANCE
+ help
+ Use the CPUFreq governor 'ondemand' as default. This allows
+ you to get a full dynamic frequency capable system by simply
+ loading your cpufreq low-level hardware driver.
+ Be aware that not all cpufreq drivers support the ondemand
+ governor. If unsure have a look at the help section of the
+ driver. Fallback governor will be the performance governor.
+
+config CPU_FREQ_DEFAULT_GOV_CONSERVATIVE
+ bool "conservative"
+ depends on !(X86_INTEL_PSTATE && SMP)
+ select CPU_FREQ_GOV_CONSERVATIVE
+ select CPU_FREQ_GOV_PERFORMANCE
+ help
+ Use the CPUFreq governor 'conservative' as default. This allows
+ you to get a full dynamic frequency capable system by simply
+ loading your cpufreq low-level hardware driver.
+ Be aware that not all cpufreq drivers support the conservative
+ governor. If unsure have a look at the help section of the
+ driver. Fallback governor will be the performance governor.
+
+config CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
+ bool "schedutil"
+ depends on SMP
+ select CPU_FREQ_GOV_SCHEDUTIL
+ select CPU_FREQ_GOV_PERFORMANCE
+ help
+ Use the 'schedutil' CPUFreq governor by default. If unsure,
+ have a look at the help section of that governor. The fallback
+ governor will be 'performance'.
+
+endchoice
+
+config CPU_FREQ_GOV_PERFORMANCE
+ tristate "'performance' governor"
+ help
+ This cpufreq governor sets the frequency statically to the
+ highest available CPU frequency.
+
+ To compile this driver as a module, choose M here: the
+ module will be called cpufreq_performance.
+
+ If in doubt, say Y.
+
+config CPU_FREQ_GOV_POWERSAVE
+ tristate "'powersave' governor"
+ help
+ This cpufreq governor sets the frequency statically to the
+ lowest available CPU frequency.
+
+ To compile this driver as a module, choose M here: the
+ module will be called cpufreq_powersave.
+
+ If in doubt, say Y.
+
+config CPU_FREQ_GOV_USERSPACE
+ tristate "'userspace' governor for userspace frequency scaling"
+ help
+ Enable this cpufreq governor when you either want to set the
+ CPU frequency manually or when a userspace program shall
+ be able to set the CPU dynamically, like on LART
+ <http://www.lartmaker.nl/>.
+
+ To compile this driver as a module, choose M here: the
+ module will be called cpufreq_userspace.
+
+ If in doubt, say Y.
+
+config CPU_FREQ_GOV_ONDEMAND
+ tristate "'ondemand' cpufreq policy governor"
+ select CPU_FREQ_GOV_COMMON
+ help
+ 'ondemand' - This driver adds a dynamic cpufreq policy governor.
+ The governor does a periodic polling and
+ changes frequency based on the CPU utilization.
+ The support for this governor depends on CPU capability to
+ do fast frequency switching (i.e, very low latency frequency
+ transitions).
+
+ To compile this driver as a module, choose M here: the
+ module will be called cpufreq_ondemand.
+
+ For details, take a look at
+ <file:Documentation/admin-guide/pm/cpufreq.rst>.
+
+ If in doubt, say N.
+
+config CPU_FREQ_GOV_CONSERVATIVE
+ tristate "'conservative' cpufreq governor"
+ depends on CPU_FREQ
+ select CPU_FREQ_GOV_COMMON
+ help
+ 'conservative' - this driver is rather similar to the 'ondemand'
+ governor both in its source code and its purpose, the difference is
+ its optimisation for better suitability in a battery powered
+ environment. The frequency is gracefully increased and decreased
+ rather than jumping to 100% when speed is required.
+
+ If you have a desktop machine then you should really be considering
+ the 'ondemand' governor instead, however if you are using a laptop,
+ PDA or even an AMD64 based computer (due to the unacceptable
+ step-by-step latency issues between the minimum and maximum frequency
+ transitions in the CPU) you will probably want to use this governor.
+
+ To compile this driver as a module, choose M here: the
+ module will be called cpufreq_conservative.
+
+ For details, take a look at
+ <file:Documentation/admin-guide/pm/cpufreq.rst>.
+
+ If in doubt, say N.
+
+config CPU_FREQ_GOV_SCHEDUTIL
+ bool "'schedutil' cpufreq policy governor"
+ depends on CPU_FREQ && SMP
+ select CPU_FREQ_GOV_ATTR_SET
+ select IRQ_WORK
+ help
+ This governor makes decisions based on the utilization data provided
+ by the scheduler. It sets the CPU frequency to be proportional to
+ the utilization/capacity ratio coming from the scheduler. If the
+ utilization is frequency-invariant, the new frequency is also
+ proportional to the maximum available frequency. If that is not the
+ case, it is proportional to the current frequency of the CPU. The
+ frequency tipping point is at utilization/capacity equal to 80% in
+ both cases.
+
+ If in doubt, say N.
+
+comment "CPU frequency scaling drivers"
+
+config CPUFREQ_DT
+ tristate "Generic DT based cpufreq driver"
+ depends on HAVE_CLK && OF
+ select CPUFREQ_DT_PLATDEV
+ select PM_OPP
+ help
+ This adds a generic DT based cpufreq driver for frequency management.
+ It supports both uniprocessor (UP) and symmetric multiprocessor (SMP)
+ systems.
+
+ If in doubt, say N.
+
+config CPUFREQ_DT_PLATDEV
+ tristate "Generic DT based cpufreq platdev driver"
+ depends on OF
+ help
+ This adds a generic DT based cpufreq platdev driver for frequency
+ management. This creates a 'cpufreq-dt' platform device, on the
+ supported platforms.
+
+ If in doubt, say N.
+
+if X86
+source "drivers/cpufreq/Kconfig.x86"
+endif
+
+if ARM || ARM64
+source "drivers/cpufreq/Kconfig.arm"
+endif
+
+if PPC32 || PPC64
+source "drivers/cpufreq/Kconfig.powerpc"
+endif
+
+if IA64
+config IA64_ACPI_CPUFREQ
+ tristate "ACPI Processor P-States driver"
+ depends on ACPI_PROCESSOR
+ help
+ This driver adds a CPUFreq driver which utilizes the ACPI
+ Processor Performance States.
+
+ If in doubt, say N.
+endif
+
+if MIPS
+config BMIPS_CPUFREQ
+ tristate "BMIPS CPUfreq Driver"
+ help
+ This option adds a CPUfreq driver for BMIPS processors with
+ support for configurable CPU frequency.
+
+ For now, BMIPS5 chips are supported (such as the Broadcom 7425).
+
+ If in doubt, say N.
+
+config LOONGSON2_CPUFREQ
+ tristate "Loongson2 CPUFreq Driver"
+ depends on LEMOTE_MACH2F
+ help
+ This option adds a CPUFreq driver for loongson processors which
+ support software configurable cpu frequency.
+
+ Loongson2F and its successors support this feature.
+
+ If in doubt, say N.
+endif
+
+if SPARC64
+config SPARC_US3_CPUFREQ
+ tristate "UltraSPARC-III CPU Frequency driver"
+ help
+ This adds the CPUFreq driver for UltraSPARC-III processors.
+
+ If in doubt, say N.
+
+config SPARC_US2E_CPUFREQ
+ tristate "UltraSPARC-IIe CPU Frequency driver"
+ help
+ This adds the CPUFreq driver for UltraSPARC-IIe processors.
+
+ If in doubt, say N.
+endif
+
+if SUPERH
+config SH_CPU_FREQ
+ tristate "SuperH CPU Frequency driver"
+ help
+ This adds the cpufreq driver for SuperH. Any CPU that supports
+ clock rate rounding through the clock framework can use this
+ driver. While it will make the kernel slightly larger, this is
+ harmless for CPUs that don't support rate rounding. The driver
+ will also generate a notice in the boot log before disabling
+ itself if the CPU in question is not capable of rate rounding.
+
+ If unsure, say N.
+endif
+
+config QORIQ_CPUFREQ
+ tristate "CPU frequency scaling driver for Freescale QorIQ SoCs"
+ depends on OF && COMMON_CLK
+ depends on PPC_E500MC || SOC_LS1021A || ARCH_LAYERSCAPE || COMPILE_TEST
+ select CLK_QORIQ
+ help
+ This adds the CPUFreq driver support for Freescale QorIQ SoCs
+ which are capable of changing the CPU's frequency dynamically.
+
+endif
+endmenu
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
new file mode 100644
index 0000000000..123b4bbfcf
--- /dev/null
+++ b/drivers/cpufreq/Kconfig.arm
@@ -0,0 +1,287 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# ARM CPU Frequency scaling drivers
+#
+
+config ACPI_CPPC_CPUFREQ
+ tristate "CPUFreq driver based on the ACPI CPPC spec"
+ depends on ACPI_PROCESSOR
+ select ACPI_CPPC_LIB
+ help
+ This adds a CPUFreq driver which uses CPPC methods
+ as described in the ACPIv5.1 spec. CPPC stands for
+ Collaborative Processor Performance Controls. It
+ is based on an abstract continuous scale of CPU
+ performance values which allows the remote power
+ processor to flexibly optimize for power and
+ performance. CPPC relies on power management firmware
+ support for its operation.
+
+ If in doubt, say N.
+
+config ACPI_CPPC_CPUFREQ_FIE
+ bool "Frequency Invariance support for CPPC cpufreq driver"
+ depends on ACPI_CPPC_CPUFREQ && GENERIC_ARCH_TOPOLOGY
+ default y
+ help
+ This extends frequency invariance support in the CPPC cpufreq driver,
+ by using CPPC delivered and reference performance counters.
+
+ If in doubt, say N.
+
+config ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM
+ tristate "Allwinner nvmem based SUN50I CPUFreq driver"
+ depends on ARCH_SUNXI
+ depends on NVMEM_SUNXI_SID
+ select PM_OPP
+ help
+ This adds the nvmem based CPUFreq driver for Allwinner
+ h6 SoC.
+
+ To compile this driver as a module, choose M here: the
+ module will be called sun50i-cpufreq-nvmem.
+
+config ARM_APPLE_SOC_CPUFREQ
+ tristate "Apple Silicon SoC CPUFreq support"
+ depends on ARCH_APPLE || (COMPILE_TEST && 64BIT)
+ select PM_OPP
+ default ARCH_APPLE
+ help
+ This adds the CPUFreq driver for Apple Silicon machines
+ (e.g. Apple M1).
+
+config ARM_ARMADA_37XX_CPUFREQ
+ tristate "Armada 37xx CPUFreq support"
+ depends on ARCH_MVEBU && CPUFREQ_DT
+ help
+ This adds the CPUFreq driver support for Marvell Armada 37xx SoCs.
+ The Armada 37xx PMU supports 4 frequency and VDD levels.
+
+config ARM_ARMADA_8K_CPUFREQ
+ tristate "Armada 8K CPUFreq driver"
+ depends on ARCH_MVEBU && CPUFREQ_DT
+ select ARMADA_AP_CPU_CLK
+ help
+ This enables the CPUFreq driver support for Marvell
+ Armada8k SOCs.
+ Armada8K device has the AP806 which supports scaling
+ to any full integer divider.
+
+ If in doubt, say N.
+
+config ARM_SCPI_CPUFREQ
+ tristate "SCPI based CPUfreq driver"
+ depends on ARM_SCPI_PROTOCOL && COMMON_CLK_SCPI
+ help
+ This adds the CPUfreq driver support for ARM platforms using SCPI
+ protocol for CPU power management.
+
+ This driver uses SCPI Message Protocol driver to interact with the
+ firmware providing the CPU DVFS functionality.
+
+config ARM_VEXPRESS_SPC_CPUFREQ
+ tristate "Versatile Express SPC based CPUfreq driver"
+ depends on ARM_CPU_TOPOLOGY && HAVE_CLK
+ depends on ARCH_VEXPRESS_SPC
+ select PM_OPP
+ help
+ This add the CPUfreq driver support for Versatile Express
+ big.LITTLE platforms using SPC for power management.
+
+config ARM_BRCMSTB_AVS_CPUFREQ
+ tristate "Broadcom STB AVS CPUfreq driver"
+ depends on ARCH_BRCMSTB || COMPILE_TEST
+ default y
+ help
+ Some Broadcom STB SoCs use a co-processor running proprietary firmware
+ ("AVS") to handle voltage and frequency scaling. This driver provides
+ a standard CPUfreq interface to the firmware.
+
+ Say Y, if you have a Broadcom SoC with AVS support for DFS or DVFS.
+
+config ARM_HIGHBANK_CPUFREQ
+ tristate "Calxeda Highbank-based"
+ depends on ARCH_HIGHBANK && CPUFREQ_DT && REGULATOR
+ default m
+ help
+ This adds the CPUFreq driver for Calxeda Highbank SoC
+ based boards.
+
+ If in doubt, say N.
+
+config ARM_IMX6Q_CPUFREQ
+ tristate "Freescale i.MX6 cpufreq support"
+ depends on ARCH_MXC
+ depends on REGULATOR_ANATOP
+ depends on NVMEM_IMX_OCOTP || COMPILE_TEST
+ select PM_OPP
+ help
+ This adds cpufreq driver support for Freescale i.MX6 series SoCs.
+
+ If in doubt, say N.
+
+config ARM_IMX_CPUFREQ_DT
+ tristate "Freescale i.MX8M cpufreq support"
+ depends on ARCH_MXC && CPUFREQ_DT
+ help
+ This adds cpufreq driver support for Freescale i.MX8M series SoCs,
+ based on cpufreq-dt.
+
+ If in doubt, say N.
+
+config ARM_KIRKWOOD_CPUFREQ
+ def_bool MACH_KIRKWOOD
+ help
+ This adds the CPUFreq driver for Marvell Kirkwood
+ SoCs.
+
+config ARM_MEDIATEK_CPUFREQ
+ tristate "CPU Frequency scaling support for MediaTek SoCs"
+ depends on ARCH_MEDIATEK && REGULATOR
+ select PM_OPP
+ help
+ This adds the CPUFreq driver support for MediaTek SoCs.
+
+config ARM_MEDIATEK_CPUFREQ_HW
+ tristate "MediaTek CPUFreq HW driver"
+ depends on ARCH_MEDIATEK || COMPILE_TEST
+ default m
+ help
+ Support for the CPUFreq HW driver.
+ Some MediaTek chipsets have a HW engine to offload the steps
+ necessary for changing the frequency of the CPUs. Firmware loaded
+ in this engine exposes a programming interface to the OS.
+ The driver implements the cpufreq interface for this HW engine.
+ Say Y if you want to support CPUFreq HW.
+
+config ARM_OMAP2PLUS_CPUFREQ
+ bool "TI OMAP2+"
+ depends on ARCH_OMAP2PLUS
+ default ARCH_OMAP2PLUS
+
+config ARM_QCOM_CPUFREQ_NVMEM
+ tristate "Qualcomm nvmem based CPUFreq"
+ depends on ARCH_QCOM
+ depends on NVMEM_QCOM_QFPROM
+ depends on QCOM_SMEM
+ select PM_OPP
+ help
+ This adds the CPUFreq driver for Qualcomm Kryo SoC based boards.
+
+ If in doubt, say N.
+
+config ARM_QCOM_CPUFREQ_HW
+ tristate "QCOM CPUFreq HW driver"
+ depends on ARCH_QCOM || COMPILE_TEST
+ help
+ Support for the CPUFreq HW driver.
+ Some QCOM chipsets have a HW engine to offload the steps
+ necessary for changing the frequency of the CPUs. Firmware loaded
+ in this engine exposes a programming interface to the OS.
+ The driver implements the cpufreq interface for this HW engine.
+ Say Y if you want to support CPUFreq HW.
+
+config ARM_RASPBERRYPI_CPUFREQ
+ tristate "Raspberry Pi cpufreq support"
+ depends on CLK_RASPBERRYPI || COMPILE_TEST
+ help
+ This adds the CPUFreq driver for Raspberry Pi
+
+ If in doubt, say N.
+
+config ARM_S3C64XX_CPUFREQ
+ bool "Samsung S3C64XX"
+ depends on CPU_S3C6410
+ default y
+ help
+ This adds the CPUFreq driver for Samsung S3C6410 SoC.
+
+ If in doubt, say N.
+
+config ARM_S5PV210_CPUFREQ
+ bool "Samsung S5PV210 and S5PC110"
+ depends on CPU_S5PV210
+ default y
+ help
+ This adds the CPUFreq driver for Samsung S5PV210 and
+ S5PC110 SoCs.
+
+ If in doubt, say N.
+
+config ARM_SA1110_CPUFREQ
+ bool
+
+config ARM_SCMI_CPUFREQ
+ tristate "SCMI based CPUfreq driver"
+ depends on ARM_SCMI_PROTOCOL || COMPILE_TEST
+ select PM_OPP
+ help
+ This adds the CPUfreq driver support for ARM platforms using SCMI
+ protocol for CPU power management.
+
+ This driver uses SCMI Message Protocol driver to interact with the
+ firmware providing the CPU DVFS functionality.
+
+config ARM_SPEAR_CPUFREQ
+ bool "SPEAr CPUFreq support"
+ depends on PLAT_SPEAR
+ default y
+ help
+ This adds the CPUFreq driver support for SPEAr SOCs.
+
+config ARM_STI_CPUFREQ
+ tristate "STi CPUFreq support"
+ depends on CPUFREQ_DT && SOC_STIH407
+ help
+ This driver uses the generic OPP framework to match the running
+ platform with a predefined set of suitable values. If not provided
+ we will fall-back so safe-values contained in Device Tree. Enable
+ this config option if you wish to add CPUFreq support for STi based
+ SoCs.
+
+config ARM_TEGRA20_CPUFREQ
+ tristate "Tegra20/30 CPUFreq support"
+ depends on ARCH_TEGRA && CPUFREQ_DT
+ default y
+ help
+ This adds the CPUFreq driver support for Tegra20/30 SOCs.
+
+config ARM_TEGRA124_CPUFREQ
+ bool "Tegra124 CPUFreq support"
+ depends on ARCH_TEGRA && CPUFREQ_DT
+ default y
+ help
+ This adds the CPUFreq driver support for Tegra124 SOCs.
+
+config ARM_TEGRA186_CPUFREQ
+ tristate "Tegra186 CPUFreq support"
+ depends on ARCH_TEGRA && TEGRA_BPMP
+ help
+ This adds the CPUFreq driver support for Tegra186 SOCs.
+
+config ARM_TEGRA194_CPUFREQ
+ tristate "Tegra194 CPUFreq support"
+ depends on ARCH_TEGRA_194_SOC && TEGRA_BPMP
+ default y
+ help
+ This adds CPU frequency driver support for Tegra194 SOCs.
+
+config ARM_TI_CPUFREQ
+ bool "Texas Instruments CPUFreq support"
+ depends on ARCH_OMAP2PLUS || ARCH_K3
+ default y
+ help
+ This driver enables valid OPPs on the running platform based on
+ values contained within the SoC in use. Enable this in order to
+ use the cpufreq-dt driver on all Texas Instruments platforms that
+ provide dt based operating-points-v2 tables with opp-supported-hw
+ data provided. Required for cpufreq support on AM335x, AM437x,
+ DRA7x, and AM57x platforms.
+
+config ARM_PXA2xx_CPUFREQ
+ tristate "Intel PXA2xx CPUfreq driver"
+ depends on PXA27x || PXA25x
+ help
+ This add the CPUFreq driver support for Intel PXA2xx SOCs.
+
+ If in doubt, say N.
diff --git a/drivers/cpufreq/Kconfig.powerpc b/drivers/cpufreq/Kconfig.powerpc
new file mode 100644
index 0000000000..58151ca566
--- /dev/null
+++ b/drivers/cpufreq/Kconfig.powerpc
@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config CPU_FREQ_CBE
+ tristate "CBE frequency scaling"
+ depends on CBE_RAS && PPC_CELL
+ default m
+ help
+ This adds the cpufreq driver for Cell BE processors.
+ For details, take a look at <file:Documentation/cpu-freq/>.
+ If you don't have such processor, say N
+
+config CPU_FREQ_CBE_PMI
+ bool "CBE frequency scaling using PMI interface"
+ depends on CPU_FREQ_CBE
+ default n
+ help
+ Select this, if you want to use the PMI interface to switch
+ frequencies. Using PMI, the processor will not only be able to run at
+ lower speed, but also at lower core voltage.
+
+config CPU_FREQ_MAPLE
+ bool "Support for Maple 970FX Evaluation Board"
+ depends on PPC_MAPLE
+ help
+ This adds support for frequency switching on Maple 970FX
+ Evaluation Board and compatible boards (IBM JS2x blades).
+
+config CPU_FREQ_PMAC
+ bool "Support for Apple PowerBooks"
+ depends on ADB_PMU && PPC32
+ help
+ This adds support for frequency switching on Apple PowerBooks,
+ this currently includes some models of iBook & Titanium
+ PowerBook.
+
+config CPU_FREQ_PMAC64
+ bool "Support for some Apple G5s"
+ depends on PPC_PMAC && PPC64
+ help
+ This adds support for frequency switching on Apple iMac G5,
+ and some of the more recent desktop G5 machines as well.
+
+config PPC_PASEMI_CPUFREQ
+ bool "Support for PA Semi PWRficient"
+ depends on PPC_PASEMI
+ default y
+ help
+ This adds the support for frequency switching on PA Semi
+ PWRficient processors.
+
+config POWERNV_CPUFREQ
+ tristate "CPU frequency scaling for IBM POWERNV platform"
+ depends on PPC_POWERNV
+ default y
+ help
+ This adds support for CPU frequency switching on IBM POWERNV
+ platform
diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86
new file mode 100644
index 0000000000..438c9e75a0
--- /dev/null
+++ b/drivers/cpufreq/Kconfig.x86
@@ -0,0 +1,341 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# x86 CPU Frequency scaling drivers
+#
+
+config X86_INTEL_PSTATE
+ bool "Intel P state control"
+ depends on X86
+ select ACPI_PROCESSOR if ACPI
+ select ACPI_CPPC_LIB if X86_64 && ACPI && SCHED_MC_PRIO
+ select CPU_FREQ_GOV_PERFORMANCE
+ select CPU_FREQ_GOV_SCHEDUTIL if SMP
+ help
+ This driver provides a P state for Intel core processors.
+ The driver implements an internal governor and will become
+ the scaling driver and governor for Sandy bridge processors.
+
+ When this driver is enabled it will become the preferred
+ scaling driver for Sandy bridge processors.
+
+ If in doubt, say N.
+
+config X86_PCC_CPUFREQ
+ tristate "Processor Clocking Control interface driver"
+ depends on ACPI && ACPI_PROCESSOR
+ help
+ This driver adds support for the PCC interface.
+
+ For details, take a look at:
+ <file:Documentation/admin-guide/pm/cpufreq_drivers.rst>.
+
+ To compile this driver as a module, choose M here: the
+ module will be called pcc-cpufreq.
+
+ If in doubt, say N.
+
+config X86_AMD_PSTATE
+ bool "AMD Processor P-State driver"
+ depends on X86 && ACPI
+ select ACPI_PROCESSOR
+ select ACPI_CPPC_LIB if X86_64
+ select CPU_FREQ_GOV_SCHEDUTIL if SMP
+ help
+ This driver adds a CPUFreq driver which utilizes a fine grain
+ processor performance frequency control range instead of legacy
+ performance levels. _CPC needs to be present in the ACPI tables
+ of the system.
+
+ For details, take a look at:
+ <file:Documentation/admin-guide/pm/amd-pstate.rst>.
+
+ If in doubt, say N.
+
+config X86_AMD_PSTATE_DEFAULT_MODE
+ int "AMD Processor P-State default mode"
+ depends on X86_AMD_PSTATE
+ default 3 if X86_AMD_PSTATE
+ range 1 4
+ help
+ Select the default mode the amd-pstate driver will use on
+ supported hardware.
+ The value set has the following meanings:
+ 1 -> Disabled
+ 2 -> Passive
+ 3 -> Active (EPP)
+ 4 -> Guided
+
+ For details, take a look at:
+ <file:Documentation/admin-guide/pm/amd-pstate.rst>.
+
+config X86_AMD_PSTATE_UT
+ tristate "selftest for AMD Processor P-State driver"
+ depends on X86 && ACPI_PROCESSOR
+ default n
+ help
+ This kernel module is used for testing. It's safe to say M here.
+
+ It can also be built-in without X86_AMD_PSTATE enabled.
+ Currently, only tests for amd-pstate are supported. If X86_AMD_PSTATE
+ is set disabled, it can tell the users test can only run on amd-pstate
+ driver, please set X86_AMD_PSTATE enabled.
+ In the future, comparison tests will be added. It can set amd-pstate
+ disabled and set acpi-cpufreq enabled to run test cases, then compare
+ the test results.
+
+config X86_ACPI_CPUFREQ
+ tristate "ACPI Processor P-States driver"
+ depends on ACPI_PROCESSOR
+ help
+ This driver adds a CPUFreq driver which utilizes the ACPI
+ Processor Performance States.
+ This driver also supports Intel Enhanced Speedstep and newer
+ AMD CPUs.
+
+ To compile this driver as a module, choose M here: the
+ module will be called acpi-cpufreq.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_ACPI_CPUFREQ_CPB
+ default y
+ bool "Legacy cpb sysfs knob support for AMD CPUs"
+ depends on X86_ACPI_CPUFREQ && CPU_SUP_AMD
+ help
+ The powernow-k8 driver used to provide a sysfs knob called "cpb"
+ to disable the Core Performance Boosting feature of AMD CPUs. This
+ file has now been superseded by the more generic "boost" entry.
+
+ By enabling this option the acpi_cpufreq driver provides the old
+ entry in addition to the new boost ones, for compatibility reasons.
+
+config ELAN_CPUFREQ
+ tristate "AMD Elan SC400 and SC410"
+ depends on MELAN
+ help
+ This adds the CPUFreq driver for AMD Elan SC400 and SC410
+ processors.
+
+ You need to specify the processor maximum speed as boot
+ parameter: elanfreq=maxspeed (in kHz) or as module
+ parameter "max_freq".
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config SC520_CPUFREQ
+ tristate "AMD Elan SC520"
+ depends on MELAN
+ help
+ This adds the CPUFreq driver for AMD Elan SC520 processor.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+
+config X86_POWERNOW_K6
+ tristate "AMD Mobile K6-2/K6-3 PowerNow!"
+ depends on X86_32
+ help
+ This adds the CPUFreq driver for mobile AMD K6-2+ and mobile
+ AMD K6-3+ processors.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_POWERNOW_K7
+ tristate "AMD Mobile Athlon/Duron PowerNow!"
+ depends on X86_32
+ help
+ This adds the CPUFreq driver for mobile AMD K7 mobile processors.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_POWERNOW_K7_ACPI
+ bool
+ depends on X86_POWERNOW_K7 && ACPI_PROCESSOR
+ depends on !(X86_POWERNOW_K7 = y && ACPI_PROCESSOR = m)
+ depends on X86_32
+ default y
+
+config X86_POWERNOW_K8
+ tristate "AMD Opteron/Athlon64 PowerNow!"
+ depends on ACPI && ACPI_PROCESSOR && X86_ACPI_CPUFREQ
+ help
+ This adds the CPUFreq driver for K8/early Opteron/Athlon64 processors.
+ Support for K10 and newer processors is now in acpi-cpufreq.
+
+ To compile this driver as a module, choose M here: the
+ module will be called powernow-k8.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+config X86_AMD_FREQ_SENSITIVITY
+ tristate "AMD frequency sensitivity feedback powersave bias"
+ depends on CPU_FREQ_GOV_ONDEMAND && X86_ACPI_CPUFREQ && CPU_SUP_AMD
+ help
+ This adds AMD-specific powersave bias function to the ondemand
+ governor, which allows it to make more power-conscious frequency
+ change decisions based on feedback from hardware (available on AMD
+ Family 16h and above).
+
+ Hardware feedback tells software how "sensitive" to frequency changes
+ the CPUs' workloads are. CPU-bound workloads will be more sensitive
+ -- they will perform better as frequency increases. Memory/IO-bound
+ workloads will be less sensitive -- they will not necessarily perform
+ better as frequency increases.
+
+ If in doubt, say N.
+
+config X86_GX_SUSPMOD
+ tristate "Cyrix MediaGX/NatSemi Geode Suspend Modulation"
+ depends on X86_32 && PCI
+ help
+ This add the CPUFreq driver for NatSemi Geode processors which
+ support suspend modulation.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_SPEEDSTEP_CENTRINO
+ tristate "Intel Enhanced SpeedStep (deprecated)"
+ select X86_SPEEDSTEP_CENTRINO_TABLE if X86_32
+ depends on X86_32 || (X86_64 && ACPI_PROCESSOR)
+ help
+ This is deprecated and this functionality is now merged into
+ acpi_cpufreq (X86_ACPI_CPUFREQ). Use that driver instead of
+ speedstep_centrino.
+ This adds the CPUFreq driver for Enhanced SpeedStep enabled
+ mobile CPUs. This means Intel Pentium M (Centrino) CPUs
+ or 64bit enabled Intel Xeons.
+
+ To compile this driver as a module, choose M here: the
+ module will be called speedstep-centrino.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_SPEEDSTEP_CENTRINO_TABLE
+ bool "Built-in tables for Banias CPUs"
+ depends on X86_32 && X86_SPEEDSTEP_CENTRINO
+ default y
+ help
+ Use built-in tables for Banias CPUs if ACPI encoding
+ is not available.
+
+ If in doubt, say N.
+
+config X86_SPEEDSTEP_ICH
+ tristate "Intel Speedstep on ICH-M chipsets (ioport interface)"
+ depends on X86_32
+ help
+ This adds the CPUFreq driver for certain mobile Intel Pentium III
+ (Coppermine), all mobile Intel Pentium III-M (Tualatin) and all
+ mobile Intel Pentium 4 P4-M on systems which have an Intel ICH2,
+ ICH3 or ICH4 southbridge.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_SPEEDSTEP_SMI
+ tristate "Intel SpeedStep on 440BX/ZX/MX chipsets (SMI interface)"
+ depends on X86_32
+ help
+ This adds the CPUFreq driver for certain mobile Intel Pentium III
+ (Coppermine), all mobile Intel Pentium III-M (Tualatin)
+ on systems which have an Intel 440BX/ZX/MX southbridge.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_P4_CLOCKMOD
+ tristate "Intel Pentium 4 clock modulation"
+ help
+ This adds the CPUFreq driver for Intel Pentium 4 / XEON
+ processors. When enabled it will lower CPU temperature by skipping
+ clocks.
+
+ This driver should be only used in exceptional
+ circumstances when very low power is needed because it causes severe
+ slowdowns and noticeable latencies. Normally Speedstep should be used
+ instead.
+
+ To compile this driver as a module, choose M here: the
+ module will be called p4-clockmod.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ Unless you are absolutely sure say N.
+
+config X86_CPUFREQ_NFORCE2
+ tristate "nVidia nForce2 FSB changing"
+ depends on X86_32
+ help
+ This adds the CPUFreq driver for FSB changing on nVidia nForce2
+ platforms.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_LONGRUN
+ tristate "Transmeta LongRun"
+ depends on X86_32
+ help
+ This adds the CPUFreq driver for Transmeta Crusoe and Efficeon processors
+ which support LongRun.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_LONGHAUL
+ tristate "VIA Cyrix III Longhaul"
+ depends on X86_32 && ACPI_PROCESSOR
+ help
+ This adds the CPUFreq driver for VIA Samuel/CyrixIII,
+ VIA Cyrix Samuel/C3, VIA Cyrix Ezra and VIA Cyrix Ezra-T
+ processors.
+
+ For details, take a look at <file:Documentation/cpu-freq/>.
+
+ If in doubt, say N.
+
+config X86_E_POWERSAVER
+ tristate "VIA C7 Enhanced PowerSaver (DANGEROUS)"
+ depends on X86_32 && ACPI_PROCESSOR
+ help
+ This adds the CPUFreq driver for VIA C7 processors. However, this driver
+ does not have any safeguards to prevent operating the CPU out of spec
+ and is thus considered dangerous. Please use the regular ACPI cpufreq
+ driver, enabled by CONFIG_X86_ACPI_CPUFREQ.
+
+ If in doubt, say N.
+
+comment "shared options"
+
+config X86_SPEEDSTEP_LIB
+ tristate
+ default (X86_SPEEDSTEP_ICH || X86_SPEEDSTEP_SMI || X86_P4_CLOCKMOD)
+
+config X86_SPEEDSTEP_RELAXED_CAP_CHECK
+ bool "Relaxed speedstep capability checks"
+ depends on X86_32 && (X86_SPEEDSTEP_SMI || X86_SPEEDSTEP_ICH)
+ help
+ Don't perform all checks for a speedstep capable system which would
+ normally be done. Some ancient or strange systems, though speedstep
+ capable, don't always indicate that they are speedstep capable. This
+ option lets the probing code bypass some of those checks if the
+ parameter "relaxed_check=1" is passed to the module.
+
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
new file mode 100644
index 0000000000..ef85107749
--- /dev/null
+++ b/drivers/cpufreq/Makefile
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: GPL-2.0
+# CPUfreq core
+obj-$(CONFIG_CPU_FREQ) += cpufreq.o freq_table.o
+
+# CPUfreq stats
+obj-$(CONFIG_CPU_FREQ_STAT) += cpufreq_stats.o
+
+# CPUfreq governors
+obj-$(CONFIG_CPU_FREQ_GOV_PERFORMANCE) += cpufreq_performance.o
+obj-$(CONFIG_CPU_FREQ_GOV_POWERSAVE) += cpufreq_powersave.o
+obj-$(CONFIG_CPU_FREQ_GOV_USERSPACE) += cpufreq_userspace.o
+obj-$(CONFIG_CPU_FREQ_GOV_ONDEMAND) += cpufreq_ondemand.o
+obj-$(CONFIG_CPU_FREQ_GOV_CONSERVATIVE) += cpufreq_conservative.o
+obj-$(CONFIG_CPU_FREQ_GOV_COMMON) += cpufreq_governor.o
+obj-$(CONFIG_CPU_FREQ_GOV_ATTR_SET) += cpufreq_governor_attr_set.o
+
+obj-$(CONFIG_CPUFREQ_DT) += cpufreq-dt.o
+obj-$(CONFIG_CPUFREQ_DT_PLATDEV) += cpufreq-dt-platdev.o
+
+# Traces
+CFLAGS_amd-pstate-trace.o := -I$(src)
+amd_pstate-y := amd-pstate.o amd-pstate-trace.o
+
+##################################################################################
+# x86 drivers.
+# Link order matters. K8 is preferred to ACPI because of firmware bugs in early
+# K8 systems. This is still the case but acpi-cpufreq errors out so that
+# powernow-k8 can load then. ACPI is preferred to all other hardware-specific drivers.
+# speedstep-* is preferred over p4-clockmod.
+
+obj-$(CONFIG_X86_ACPI_CPUFREQ) += acpi-cpufreq.o
+obj-$(CONFIG_X86_AMD_PSTATE) += amd_pstate.o
+obj-$(CONFIG_X86_AMD_PSTATE_UT) += amd-pstate-ut.o
+obj-$(CONFIG_X86_POWERNOW_K8) += powernow-k8.o
+obj-$(CONFIG_X86_PCC_CPUFREQ) += pcc-cpufreq.o
+obj-$(CONFIG_X86_POWERNOW_K6) += powernow-k6.o
+obj-$(CONFIG_X86_POWERNOW_K7) += powernow-k7.o
+obj-$(CONFIG_X86_LONGHAUL) += longhaul.o
+obj-$(CONFIG_X86_E_POWERSAVER) += e_powersaver.o
+obj-$(CONFIG_ELAN_CPUFREQ) += elanfreq.o
+obj-$(CONFIG_SC520_CPUFREQ) += sc520_freq.o
+obj-$(CONFIG_X86_LONGRUN) += longrun.o
+obj-$(CONFIG_X86_GX_SUSPMOD) += gx-suspmod.o
+obj-$(CONFIG_X86_SPEEDSTEP_ICH) += speedstep-ich.o
+obj-$(CONFIG_X86_SPEEDSTEP_LIB) += speedstep-lib.o
+obj-$(CONFIG_X86_SPEEDSTEP_SMI) += speedstep-smi.o
+obj-$(CONFIG_X86_SPEEDSTEP_CENTRINO) += speedstep-centrino.o
+obj-$(CONFIG_X86_P4_CLOCKMOD) += p4-clockmod.o
+obj-$(CONFIG_X86_CPUFREQ_NFORCE2) += cpufreq-nforce2.o
+obj-$(CONFIG_X86_INTEL_PSTATE) += intel_pstate.o
+obj-$(CONFIG_X86_AMD_FREQ_SENSITIVITY) += amd_freq_sensitivity.o
+
+##################################################################################
+# ARM SoC drivers
+obj-$(CONFIG_ARM_APPLE_SOC_CPUFREQ) += apple-soc-cpufreq.o
+obj-$(CONFIG_ARM_ARMADA_37XX_CPUFREQ) += armada-37xx-cpufreq.o
+obj-$(CONFIG_ARM_ARMADA_8K_CPUFREQ) += armada-8k-cpufreq.o
+obj-$(CONFIG_ARM_BRCMSTB_AVS_CPUFREQ) += brcmstb-avs-cpufreq.o
+obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o
+obj-$(CONFIG_ARCH_DAVINCI) += davinci-cpufreq.o
+obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ) += highbank-cpufreq.o
+obj-$(CONFIG_ARM_IMX6Q_CPUFREQ) += imx6q-cpufreq.o
+obj-$(CONFIG_ARM_IMX_CPUFREQ_DT) += imx-cpufreq-dt.o
+obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ) += kirkwood-cpufreq.o
+obj-$(CONFIG_ARM_MEDIATEK_CPUFREQ) += mediatek-cpufreq.o
+obj-$(CONFIG_ARM_MEDIATEK_CPUFREQ_HW) += mediatek-cpufreq-hw.o
+obj-$(CONFIG_MACH_MVEBU_V7) += mvebu-cpufreq.o
+obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ) += omap-cpufreq.o
+obj-$(CONFIG_ARM_PXA2xx_CPUFREQ) += pxa2xx-cpufreq.o
+obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o
+obj-$(CONFIG_ARM_QCOM_CPUFREQ_HW) += qcom-cpufreq-hw.o
+obj-$(CONFIG_ARM_QCOM_CPUFREQ_NVMEM) += qcom-cpufreq-nvmem.o
+obj-$(CONFIG_ARM_RASPBERRYPI_CPUFREQ) += raspberrypi-cpufreq.o
+obj-$(CONFIG_ARM_S3C64XX_CPUFREQ) += s3c64xx-cpufreq.o
+obj-$(CONFIG_ARM_S5PV210_CPUFREQ) += s5pv210-cpufreq.o
+obj-$(CONFIG_ARM_SA1110_CPUFREQ) += sa1110-cpufreq.o
+obj-$(CONFIG_ARM_SCMI_CPUFREQ) += scmi-cpufreq.o
+obj-$(CONFIG_ARM_SCPI_CPUFREQ) += scpi-cpufreq.o
+obj-$(CONFIG_ARM_SPEAR_CPUFREQ) += spear-cpufreq.o
+obj-$(CONFIG_ARM_STI_CPUFREQ) += sti-cpufreq.o
+obj-$(CONFIG_ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM) += sun50i-cpufreq-nvmem.o
+obj-$(CONFIG_ARM_TEGRA20_CPUFREQ) += tegra20-cpufreq.o
+obj-$(CONFIG_ARM_TEGRA124_CPUFREQ) += tegra124-cpufreq.o
+obj-$(CONFIG_ARM_TEGRA186_CPUFREQ) += tegra186-cpufreq.o
+obj-$(CONFIG_ARM_TEGRA194_CPUFREQ) += tegra194-cpufreq.o
+obj-$(CONFIG_ARM_TI_CPUFREQ) += ti-cpufreq.o
+obj-$(CONFIG_ARM_VEXPRESS_SPC_CPUFREQ) += vexpress-spc-cpufreq.o
+
+
+##################################################################################
+# PowerPC platform drivers
+obj-$(CONFIG_CPU_FREQ_CBE) += ppc-cbe-cpufreq.o
+ppc-cbe-cpufreq-y += ppc_cbe_cpufreq_pervasive.o ppc_cbe_cpufreq.o
+obj-$(CONFIG_CPU_FREQ_CBE_PMI) += ppc_cbe_cpufreq_pmi.o
+obj-$(CONFIG_CPU_FREQ_MAPLE) += maple-cpufreq.o
+obj-$(CONFIG_QORIQ_CPUFREQ) += qoriq-cpufreq.o
+obj-$(CONFIG_CPU_FREQ_PMAC) += pmac32-cpufreq.o
+obj-$(CONFIG_CPU_FREQ_PMAC64) += pmac64-cpufreq.o
+obj-$(CONFIG_PPC_PASEMI_CPUFREQ) += pasemi-cpufreq.o
+obj-$(CONFIG_POWERNV_CPUFREQ) += powernv-cpufreq.o
+
+##################################################################################
+# Other platform drivers
+obj-$(CONFIG_BMIPS_CPUFREQ) += bmips-cpufreq.o
+obj-$(CONFIG_IA64_ACPI_CPUFREQ) += ia64-acpi-cpufreq.o
+obj-$(CONFIG_LOONGSON2_CPUFREQ) += loongson2_cpufreq.o
+obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o
+obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o
+obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
new file mode 100644
index 0000000000..37f1cdf46d
--- /dev/null
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -0,0 +1,1048 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * acpi-cpufreq.c - ACPI Processor P-States Driver
+ *
+ * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
+ * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
+ * Copyright (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de>
+ * Copyright (C) 2006 Denis Sadykov <denis.m.sadykov@intel.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/cpufreq.h>
+#include <linux/compiler.h>
+#include <linux/dmi.h>
+#include <linux/slab.h>
+#include <linux/string_helpers.h>
+#include <linux/platform_device.h>
+
+#include <linux/acpi.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/uaccess.h>
+
+#include <acpi/processor.h>
+#include <acpi/cppc_acpi.h>
+
+#include <asm/msr.h>
+#include <asm/processor.h>
+#include <asm/cpufeature.h>
+#include <asm/cpu_device_id.h>
+
+MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski");
+MODULE_DESCRIPTION("ACPI Processor P-States Driver");
+MODULE_LICENSE("GPL");
+
+enum {
+ UNDEFINED_CAPABLE = 0,
+ SYSTEM_INTEL_MSR_CAPABLE,
+ SYSTEM_AMD_MSR_CAPABLE,
+ SYSTEM_IO_CAPABLE,
+};
+
+#define INTEL_MSR_RANGE (0xffff)
+#define AMD_MSR_RANGE (0x7)
+#define HYGON_MSR_RANGE (0x7)
+
+#define MSR_K7_HWCR_CPB_DIS (1ULL << 25)
+
+struct acpi_cpufreq_data {
+ unsigned int resume;
+ unsigned int cpu_feature;
+ unsigned int acpi_perf_cpu;
+ cpumask_var_t freqdomain_cpus;
+ void (*cpu_freq_write)(struct acpi_pct_register *reg, u32 val);
+ u32 (*cpu_freq_read)(struct acpi_pct_register *reg);
+};
+
+/* acpi_perf_data is a pointer to percpu data. */
+static struct acpi_processor_performance __percpu *acpi_perf_data;
+
+static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufreq_data *data)
+{
+ return per_cpu_ptr(acpi_perf_data, data->acpi_perf_cpu);
+}
+
+static struct cpufreq_driver acpi_cpufreq_driver;
+
+static unsigned int acpi_pstate_strict;
+
+static bool boost_state(unsigned int cpu)
+{
+ u32 lo, hi;
+ u64 msr;
+
+ switch (boot_cpu_data.x86_vendor) {
+ case X86_VENDOR_INTEL:
+ case X86_VENDOR_CENTAUR:
+ case X86_VENDOR_ZHAOXIN:
+ rdmsr_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &lo, &hi);
+ msr = lo | ((u64)hi << 32);
+ return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
+ case X86_VENDOR_HYGON:
+ case X86_VENDOR_AMD:
+ rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi);
+ msr = lo | ((u64)hi << 32);
+ return !(msr & MSR_K7_HWCR_CPB_DIS);
+ }
+ return false;
+}
+
+static int boost_set_msr(bool enable)
+{
+ u32 msr_addr;
+ u64 msr_mask, val;
+
+ switch (boot_cpu_data.x86_vendor) {
+ case X86_VENDOR_INTEL:
+ case X86_VENDOR_CENTAUR:
+ case X86_VENDOR_ZHAOXIN:
+ msr_addr = MSR_IA32_MISC_ENABLE;
+ msr_mask = MSR_IA32_MISC_ENABLE_TURBO_DISABLE;
+ break;
+ case X86_VENDOR_HYGON:
+ case X86_VENDOR_AMD:
+ msr_addr = MSR_K7_HWCR;
+ msr_mask = MSR_K7_HWCR_CPB_DIS;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ rdmsrl(msr_addr, val);
+
+ if (enable)
+ val &= ~msr_mask;
+ else
+ val |= msr_mask;
+
+ wrmsrl(msr_addr, val);
+ return 0;
+}
+
+static void boost_set_msr_each(void *p_en)
+{
+ bool enable = (bool) p_en;
+
+ boost_set_msr(enable);
+}
+
+static int set_boost(struct cpufreq_policy *policy, int val)
+{
+ on_each_cpu_mask(policy->cpus, boost_set_msr_each,
+ (void *)(long)val, 1);
+ pr_debug("CPU %*pbl: Core Boosting %s.\n",
+ cpumask_pr_args(policy->cpus), str_enabled_disabled(val));
+
+ return 0;
+}
+
+static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+
+ if (unlikely(!data))
+ return -ENODEV;
+
+ return cpufreq_show_cpus(data->freqdomain_cpus, buf);
+}
+
+cpufreq_freq_attr_ro(freqdomain_cpus);
+
+#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
+static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf,
+ size_t count)
+{
+ int ret;
+ unsigned int val = 0;
+
+ if (!acpi_cpufreq_driver.set_boost)
+ return -EINVAL;
+
+ ret = kstrtouint(buf, 10, &val);
+ if (ret || val > 1)
+ return -EINVAL;
+
+ cpus_read_lock();
+ set_boost(policy, val);
+ cpus_read_unlock();
+
+ return count;
+}
+
+static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf)
+{
+ return sprintf(buf, "%u\n", acpi_cpufreq_driver.boost_enabled);
+}
+
+cpufreq_freq_attr_rw(cpb);
+#endif
+
+static int check_est_cpu(unsigned int cpuid)
+{
+ struct cpuinfo_x86 *cpu = &cpu_data(cpuid);
+
+ return cpu_has(cpu, X86_FEATURE_EST);
+}
+
+static int check_amd_hwpstate_cpu(unsigned int cpuid)
+{
+ struct cpuinfo_x86 *cpu = &cpu_data(cpuid);
+
+ return cpu_has(cpu, X86_FEATURE_HW_PSTATE);
+}
+
+static unsigned extract_io(struct cpufreq_policy *policy, u32 value)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+ struct acpi_processor_performance *perf;
+ int i;
+
+ perf = to_perf_data(data);
+
+ for (i = 0; i < perf->state_count; i++) {
+ if (value == perf->states[i].status)
+ return policy->freq_table[i].frequency;
+ }
+ return 0;
+}
+
+static unsigned extract_msr(struct cpufreq_policy *policy, u32 msr)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+ struct cpufreq_frequency_table *pos;
+ struct acpi_processor_performance *perf;
+
+ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
+ msr &= AMD_MSR_RANGE;
+ else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
+ msr &= HYGON_MSR_RANGE;
+ else
+ msr &= INTEL_MSR_RANGE;
+
+ perf = to_perf_data(data);
+
+ cpufreq_for_each_entry(pos, policy->freq_table)
+ if (msr == perf->states[pos->driver_data].status)
+ return pos->frequency;
+ return policy->freq_table[0].frequency;
+}
+
+static unsigned extract_freq(struct cpufreq_policy *policy, u32 val)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+
+ switch (data->cpu_feature) {
+ case SYSTEM_INTEL_MSR_CAPABLE:
+ case SYSTEM_AMD_MSR_CAPABLE:
+ return extract_msr(policy, val);
+ case SYSTEM_IO_CAPABLE:
+ return extract_io(policy, val);
+ default:
+ return 0;
+ }
+}
+
+static u32 cpu_freq_read_intel(struct acpi_pct_register *not_used)
+{
+ u32 val, dummy __always_unused;
+
+ rdmsr(MSR_IA32_PERF_CTL, val, dummy);
+ return val;
+}
+
+static void cpu_freq_write_intel(struct acpi_pct_register *not_used, u32 val)
+{
+ u32 lo, hi;
+
+ rdmsr(MSR_IA32_PERF_CTL, lo, hi);
+ lo = (lo & ~INTEL_MSR_RANGE) | (val & INTEL_MSR_RANGE);
+ wrmsr(MSR_IA32_PERF_CTL, lo, hi);
+}
+
+static u32 cpu_freq_read_amd(struct acpi_pct_register *not_used)
+{
+ u32 val, dummy __always_unused;
+
+ rdmsr(MSR_AMD_PERF_CTL, val, dummy);
+ return val;
+}
+
+static void cpu_freq_write_amd(struct acpi_pct_register *not_used, u32 val)
+{
+ wrmsr(MSR_AMD_PERF_CTL, val, 0);
+}
+
+static u32 cpu_freq_read_io(struct acpi_pct_register *reg)
+{
+ u32 val;
+
+ acpi_os_read_port(reg->address, &val, reg->bit_width);
+ return val;
+}
+
+static void cpu_freq_write_io(struct acpi_pct_register *reg, u32 val)
+{
+ acpi_os_write_port(reg->address, val, reg->bit_width);
+}
+
+struct drv_cmd {
+ struct acpi_pct_register *reg;
+ u32 val;
+ union {
+ void (*write)(struct acpi_pct_register *reg, u32 val);
+ u32 (*read)(struct acpi_pct_register *reg);
+ } func;
+};
+
+/* Called via smp_call_function_single(), on the target CPU */
+static void do_drv_read(void *_cmd)
+{
+ struct drv_cmd *cmd = _cmd;
+
+ cmd->val = cmd->func.read(cmd->reg);
+}
+
+static u32 drv_read(struct acpi_cpufreq_data *data, const struct cpumask *mask)
+{
+ struct acpi_processor_performance *perf = to_perf_data(data);
+ struct drv_cmd cmd = {
+ .reg = &perf->control_register,
+ .func.read = data->cpu_freq_read,
+ };
+ int err;
+
+ err = smp_call_function_any(mask, do_drv_read, &cmd, 1);
+ WARN_ON_ONCE(err); /* smp_call_function_any() was buggy? */
+ return cmd.val;
+}
+
+/* Called via smp_call_function_many(), on the target CPUs */
+static void do_drv_write(void *_cmd)
+{
+ struct drv_cmd *cmd = _cmd;
+
+ cmd->func.write(cmd->reg, cmd->val);
+}
+
+static void drv_write(struct acpi_cpufreq_data *data,
+ const struct cpumask *mask, u32 val)
+{
+ struct acpi_processor_performance *perf = to_perf_data(data);
+ struct drv_cmd cmd = {
+ .reg = &perf->control_register,
+ .val = val,
+ .func.write = data->cpu_freq_write,
+ };
+ int this_cpu;
+
+ this_cpu = get_cpu();
+ if (cpumask_test_cpu(this_cpu, mask))
+ do_drv_write(&cmd);
+
+ smp_call_function_many(mask, do_drv_write, &cmd, 1);
+ put_cpu();
+}
+
+static u32 get_cur_val(const struct cpumask *mask, struct acpi_cpufreq_data *data)
+{
+ u32 val;
+
+ if (unlikely(cpumask_empty(mask)))
+ return 0;
+
+ val = drv_read(data, mask);
+
+ pr_debug("%s = %u\n", __func__, val);
+
+ return val;
+}
+
+static unsigned int get_cur_freq_on_cpu(unsigned int cpu)
+{
+ struct acpi_cpufreq_data *data;
+ struct cpufreq_policy *policy;
+ unsigned int freq;
+ unsigned int cached_freq;
+
+ pr_debug("%s (%d)\n", __func__, cpu);
+
+ policy = cpufreq_cpu_get_raw(cpu);
+ if (unlikely(!policy))
+ return 0;
+
+ data = policy->driver_data;
+ if (unlikely(!data || !policy->freq_table))
+ return 0;
+
+ cached_freq = policy->freq_table[to_perf_data(data)->state].frequency;
+ freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data));
+ if (freq != cached_freq) {
+ /*
+ * The dreaded BIOS frequency change behind our back.
+ * Force set the frequency on next target call.
+ */
+ data->resume = 1;
+ }
+
+ pr_debug("cur freq = %u\n", freq);
+
+ return freq;
+}
+
+static unsigned int check_freqs(struct cpufreq_policy *policy,
+ const struct cpumask *mask, unsigned int freq)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+ unsigned int cur_freq;
+ unsigned int i;
+
+ for (i = 0; i < 100; i++) {
+ cur_freq = extract_freq(policy, get_cur_val(mask, data));
+ if (cur_freq == freq)
+ return 1;
+ udelay(10);
+ }
+ return 0;
+}
+
+static int acpi_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+ struct acpi_processor_performance *perf;
+ const struct cpumask *mask;
+ unsigned int next_perf_state = 0; /* Index into perf table */
+ int result = 0;
+
+ if (unlikely(!data)) {
+ return -ENODEV;
+ }
+
+ perf = to_perf_data(data);
+ next_perf_state = policy->freq_table[index].driver_data;
+ if (perf->state == next_perf_state) {
+ if (unlikely(data->resume)) {
+ pr_debug("Called after resume, resetting to P%d\n",
+ next_perf_state);
+ data->resume = 0;
+ } else {
+ pr_debug("Already at target state (P%d)\n",
+ next_perf_state);
+ return 0;
+ }
+ }
+
+ /*
+ * The core won't allow CPUs to go away until the governor has been
+ * stopped, so we can rely on the stability of policy->cpus.
+ */
+ mask = policy->shared_type == CPUFREQ_SHARED_TYPE_ANY ?
+ cpumask_of(policy->cpu) : policy->cpus;
+
+ drv_write(data, mask, perf->states[next_perf_state].control);
+
+ if (acpi_pstate_strict) {
+ if (!check_freqs(policy, mask,
+ policy->freq_table[index].frequency)) {
+ pr_debug("%s (%d)\n", __func__, policy->cpu);
+ result = -EAGAIN;
+ }
+ }
+
+ if (!result)
+ perf->state = next_perf_state;
+
+ return result;
+}
+
+static unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+ struct acpi_processor_performance *perf;
+ struct cpufreq_frequency_table *entry;
+ unsigned int next_perf_state, next_freq, index;
+
+ /*
+ * Find the closest frequency above target_freq.
+ */
+ if (policy->cached_target_freq == target_freq)
+ index = policy->cached_resolved_idx;
+ else
+ index = cpufreq_table_find_index_dl(policy, target_freq,
+ false);
+
+ entry = &policy->freq_table[index];
+ next_freq = entry->frequency;
+ next_perf_state = entry->driver_data;
+
+ perf = to_perf_data(data);
+ if (perf->state == next_perf_state) {
+ if (unlikely(data->resume))
+ data->resume = 0;
+ else
+ return next_freq;
+ }
+
+ data->cpu_freq_write(&perf->control_register,
+ perf->states[next_perf_state].control);
+ perf->state = next_perf_state;
+ return next_freq;
+}
+
+static unsigned long
+acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu)
+{
+ struct acpi_processor_performance *perf;
+
+ perf = to_perf_data(data);
+ if (cpu_khz) {
+ /* search the closest match to cpu_khz */
+ unsigned int i;
+ unsigned long freq;
+ unsigned long freqn = perf->states[0].core_frequency * 1000;
+
+ for (i = 0; i < (perf->state_count-1); i++) {
+ freq = freqn;
+ freqn = perf->states[i+1].core_frequency * 1000;
+ if ((2 * cpu_khz) > (freqn + freq)) {
+ perf->state = i;
+ return freq;
+ }
+ }
+ perf->state = perf->state_count-1;
+ return freqn;
+ } else {
+ /* assume CPU is at P0... */
+ perf->state = 0;
+ return perf->states[0].core_frequency * 1000;
+ }
+}
+
+static void free_acpi_perf_data(void)
+{
+ unsigned int i;
+
+ /* Freeing a NULL pointer is OK, and alloc_percpu zeroes. */
+ for_each_possible_cpu(i)
+ free_cpumask_var(per_cpu_ptr(acpi_perf_data, i)
+ ->shared_cpu_map);
+ free_percpu(acpi_perf_data);
+}
+
+static int cpufreq_boost_down_prep(unsigned int cpu)
+{
+ /*
+ * Clear the boost-disable bit on the CPU_DOWN path so that
+ * this cpu cannot block the remaining ones from boosting.
+ */
+ return boost_set_msr(1);
+}
+
+/*
+ * acpi_cpufreq_early_init - initialize ACPI P-States library
+ *
+ * Initialize the ACPI P-States library (drivers/acpi/processor_perflib.c)
+ * in order to determine correct frequency and voltage pairings. We can
+ * do _PDC and _PSD and find out the processor dependency for the
+ * actual init that will happen later...
+ */
+static int __init acpi_cpufreq_early_init(void)
+{
+ unsigned int i;
+ pr_debug("%s\n", __func__);
+
+ acpi_perf_data = alloc_percpu(struct acpi_processor_performance);
+ if (!acpi_perf_data) {
+ pr_debug("Memory allocation error for acpi_perf_data.\n");
+ return -ENOMEM;
+ }
+ for_each_possible_cpu(i) {
+ if (!zalloc_cpumask_var_node(
+ &per_cpu_ptr(acpi_perf_data, i)->shared_cpu_map,
+ GFP_KERNEL, cpu_to_node(i))) {
+
+ /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */
+ free_acpi_perf_data();
+ return -ENOMEM;
+ }
+ }
+
+ /* Do initialization in ACPI core */
+ acpi_processor_preregister_performance(acpi_perf_data);
+ return 0;
+}
+
+#ifdef CONFIG_SMP
+/*
+ * Some BIOSes do SW_ANY coordination internally, either set it up in hw
+ * or do it in BIOS firmware and won't inform about it to OS. If not
+ * detected, this has a side effect of making CPU run at a different speed
+ * than OS intended it to run at. Detect it and handle it cleanly.
+ */
+static int bios_with_sw_any_bug;
+
+static int sw_any_bug_found(const struct dmi_system_id *d)
+{
+ bios_with_sw_any_bug = 1;
+ return 0;
+}
+
+static const struct dmi_system_id sw_any_bug_dmi_table[] = {
+ {
+ .callback = sw_any_bug_found,
+ .ident = "Supermicro Server X6DLP",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "Supermicro"),
+ DMI_MATCH(DMI_BIOS_VERSION, "080010"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "X6DLP"),
+ },
+ },
+ { }
+};
+
+static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c)
+{
+ /* Intel Xeon Processor 7100 Series Specification Update
+ * https://www.intel.com/Assets/PDF/specupdate/314554.pdf
+ * AL30: A Machine Check Exception (MCE) Occurring during an
+ * Enhanced Intel SpeedStep Technology Ratio Change May Cause
+ * Both Processor Cores to Lock Up. */
+ if (c->x86_vendor == X86_VENDOR_INTEL) {
+ if ((c->x86 == 15) &&
+ (c->x86_model == 6) &&
+ (c->x86_stepping == 8)) {
+ pr_info("Intel(R) Xeon(R) 7100 Errata AL30, processors may lock up on frequency changes: disabling acpi-cpufreq\n");
+ return -ENODEV;
+ }
+ }
+ return 0;
+}
+#endif
+
+#ifdef CONFIG_ACPI_CPPC_LIB
+static u64 get_max_boost_ratio(unsigned int cpu)
+{
+ struct cppc_perf_caps perf_caps;
+ u64 highest_perf, nominal_perf;
+ int ret;
+
+ if (acpi_pstate_strict)
+ return 0;
+
+ ret = cppc_get_perf_caps(cpu, &perf_caps);
+ if (ret) {
+ pr_debug("CPU%d: Unable to get performance capabilities (%d)\n",
+ cpu, ret);
+ return 0;
+ }
+
+ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
+ highest_perf = amd_get_highest_perf();
+ else
+ highest_perf = perf_caps.highest_perf;
+
+ nominal_perf = perf_caps.nominal_perf;
+
+ if (!highest_perf || !nominal_perf) {
+ pr_debug("CPU%d: highest or nominal performance missing\n", cpu);
+ return 0;
+ }
+
+ if (highest_perf < nominal_perf) {
+ pr_debug("CPU%d: nominal performance above highest\n", cpu);
+ return 0;
+ }
+
+ return div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf);
+}
+#else
+static inline u64 get_max_boost_ratio(unsigned int cpu) { return 0; }
+#endif
+
+static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *freq_table;
+ struct acpi_processor_performance *perf;
+ struct acpi_cpufreq_data *data;
+ unsigned int cpu = policy->cpu;
+ struct cpuinfo_x86 *c = &cpu_data(cpu);
+ unsigned int valid_states = 0;
+ unsigned int result = 0;
+ u64 max_boost_ratio;
+ unsigned int i;
+#ifdef CONFIG_SMP
+ static int blacklisted;
+#endif
+
+ pr_debug("%s\n", __func__);
+
+#ifdef CONFIG_SMP
+ if (blacklisted)
+ return blacklisted;
+ blacklisted = acpi_cpufreq_blacklist(c);
+ if (blacklisted)
+ return blacklisted;
+#endif
+
+ data = kzalloc(sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ if (!zalloc_cpumask_var(&data->freqdomain_cpus, GFP_KERNEL)) {
+ result = -ENOMEM;
+ goto err_free;
+ }
+
+ perf = per_cpu_ptr(acpi_perf_data, cpu);
+ data->acpi_perf_cpu = cpu;
+ policy->driver_data = data;
+
+ if (cpu_has(c, X86_FEATURE_CONSTANT_TSC))
+ acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS;
+
+ result = acpi_processor_register_performance(perf, cpu);
+ if (result)
+ goto err_free_mask;
+
+ policy->shared_type = perf->shared_type;
+
+ /*
+ * Will let policy->cpus know about dependency only when software
+ * coordination is required.
+ */
+ if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||
+ policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {
+ cpumask_copy(policy->cpus, perf->shared_cpu_map);
+ }
+ cpumask_copy(data->freqdomain_cpus, perf->shared_cpu_map);
+
+#ifdef CONFIG_SMP
+ dmi_check_system(sw_any_bug_dmi_table);
+ if (bios_with_sw_any_bug && !policy_is_shared(policy)) {
+ policy->shared_type = CPUFREQ_SHARED_TYPE_ALL;
+ cpumask_copy(policy->cpus, topology_core_cpumask(cpu));
+ }
+
+ if (check_amd_hwpstate_cpu(cpu) && boot_cpu_data.x86 < 0x19 &&
+ !acpi_pstate_strict) {
+ cpumask_clear(policy->cpus);
+ cpumask_set_cpu(cpu, policy->cpus);
+ cpumask_copy(data->freqdomain_cpus,
+ topology_sibling_cpumask(cpu));
+ policy->shared_type = CPUFREQ_SHARED_TYPE_HW;
+ pr_info_once("overriding BIOS provided _PSD data\n");
+ }
+#endif
+
+ /* capability check */
+ if (perf->state_count <= 1) {
+ pr_debug("No P-States\n");
+ result = -ENODEV;
+ goto err_unreg;
+ }
+
+ if (perf->control_register.space_id != perf->status_register.space_id) {
+ result = -ENODEV;
+ goto err_unreg;
+ }
+
+ switch (perf->control_register.space_id) {
+ case ACPI_ADR_SPACE_SYSTEM_IO:
+ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
+ boot_cpu_data.x86 == 0xf) {
+ pr_debug("AMD K8 systems must use native drivers.\n");
+ result = -ENODEV;
+ goto err_unreg;
+ }
+ pr_debug("SYSTEM IO addr space\n");
+ data->cpu_feature = SYSTEM_IO_CAPABLE;
+ data->cpu_freq_read = cpu_freq_read_io;
+ data->cpu_freq_write = cpu_freq_write_io;
+ break;
+ case ACPI_ADR_SPACE_FIXED_HARDWARE:
+ pr_debug("HARDWARE addr space\n");
+ if (check_est_cpu(cpu)) {
+ data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE;
+ data->cpu_freq_read = cpu_freq_read_intel;
+ data->cpu_freq_write = cpu_freq_write_intel;
+ break;
+ }
+ if (check_amd_hwpstate_cpu(cpu)) {
+ data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE;
+ data->cpu_freq_read = cpu_freq_read_amd;
+ data->cpu_freq_write = cpu_freq_write_amd;
+ break;
+ }
+ result = -ENODEV;
+ goto err_unreg;
+ default:
+ pr_debug("Unknown addr space %d\n",
+ (u32) (perf->control_register.space_id));
+ result = -ENODEV;
+ goto err_unreg;
+ }
+
+ freq_table = kcalloc(perf->state_count + 1, sizeof(*freq_table),
+ GFP_KERNEL);
+ if (!freq_table) {
+ result = -ENOMEM;
+ goto err_unreg;
+ }
+
+ /* detect transition latency */
+ policy->cpuinfo.transition_latency = 0;
+ for (i = 0; i < perf->state_count; i++) {
+ if ((perf->states[i].transition_latency * 1000) >
+ policy->cpuinfo.transition_latency)
+ policy->cpuinfo.transition_latency =
+ perf->states[i].transition_latency * 1000;
+ }
+
+ /* Check for high latency (>20uS) from buggy BIOSes, like on T42 */
+ if (perf->control_register.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE &&
+ policy->cpuinfo.transition_latency > 20 * 1000) {
+ policy->cpuinfo.transition_latency = 20 * 1000;
+ pr_info_once("P-state transition latency capped at 20 uS\n");
+ }
+
+ /* table init */
+ for (i = 0; i < perf->state_count; i++) {
+ if (i > 0 && perf->states[i].core_frequency >=
+ freq_table[valid_states-1].frequency / 1000)
+ continue;
+
+ freq_table[valid_states].driver_data = i;
+ freq_table[valid_states].frequency =
+ perf->states[i].core_frequency * 1000;
+ valid_states++;
+ }
+ freq_table[valid_states].frequency = CPUFREQ_TABLE_END;
+
+ max_boost_ratio = get_max_boost_ratio(cpu);
+ if (max_boost_ratio) {
+ unsigned int freq = freq_table[0].frequency;
+
+ /*
+ * Because the loop above sorts the freq_table entries in the
+ * descending order, freq is the maximum frequency in the table.
+ * Assume that it corresponds to the CPPC nominal frequency and
+ * use it to set cpuinfo.max_freq.
+ */
+ policy->cpuinfo.max_freq = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT;
+ } else {
+ /*
+ * If the maximum "boost" frequency is unknown, ask the arch
+ * scale-invariance code to use the "nominal" performance for
+ * CPU utilization scaling so as to prevent the schedutil
+ * governor from selecting inadequate CPU frequencies.
+ */
+ arch_set_max_freq_ratio(true);
+ }
+
+ policy->freq_table = freq_table;
+ perf->state = 0;
+
+ switch (perf->control_register.space_id) {
+ case ACPI_ADR_SPACE_SYSTEM_IO:
+ /*
+ * The core will not set policy->cur, because
+ * cpufreq_driver->get is NULL, so we need to set it here.
+ * However, we have to guess it, because the current speed is
+ * unknown and not detectable via IO ports.
+ */
+ policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu);
+ break;
+ case ACPI_ADR_SPACE_FIXED_HARDWARE:
+ acpi_cpufreq_driver.get = get_cur_freq_on_cpu;
+ break;
+ default:
+ break;
+ }
+
+ /* notify BIOS that we exist */
+ acpi_processor_notify_smm(THIS_MODULE);
+
+ pr_debug("CPU%u - ACPI performance management activated.\n", cpu);
+ for (i = 0; i < perf->state_count; i++)
+ pr_debug(" %cP%d: %d MHz, %d mW, %d uS\n",
+ (i == perf->state ? '*' : ' '), i,
+ (u32) perf->states[i].core_frequency,
+ (u32) perf->states[i].power,
+ (u32) perf->states[i].transition_latency);
+
+ /*
+ * the first call to ->target() should result in us actually
+ * writing something to the appropriate registers.
+ */
+ data->resume = 1;
+
+ policy->fast_switch_possible = !acpi_pstate_strict &&
+ !(policy_is_shared(policy) && policy->shared_type != CPUFREQ_SHARED_TYPE_ANY);
+
+ if (perf->states[0].core_frequency * 1000 != freq_table[0].frequency)
+ pr_warn(FW_WARN "P-state 0 is not max freq\n");
+
+ if (acpi_cpufreq_driver.set_boost)
+ set_boost(policy, acpi_cpufreq_driver.boost_enabled);
+
+ return result;
+
+err_unreg:
+ acpi_processor_unregister_performance(cpu);
+err_free_mask:
+ free_cpumask_var(data->freqdomain_cpus);
+err_free:
+ kfree(data);
+ policy->driver_data = NULL;
+
+ return result;
+}
+
+static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+
+ pr_debug("%s\n", __func__);
+
+ cpufreq_boost_down_prep(policy->cpu);
+ policy->fast_switch_possible = false;
+ policy->driver_data = NULL;
+ acpi_processor_unregister_performance(data->acpi_perf_cpu);
+ free_cpumask_var(data->freqdomain_cpus);
+ kfree(policy->freq_table);
+ kfree(data);
+
+ return 0;
+}
+
+static int acpi_cpufreq_resume(struct cpufreq_policy *policy)
+{
+ struct acpi_cpufreq_data *data = policy->driver_data;
+
+ pr_debug("%s\n", __func__);
+
+ data->resume = 1;
+
+ return 0;
+}
+
+static struct freq_attr *acpi_cpufreq_attr[] = {
+ &cpufreq_freq_attr_scaling_available_freqs,
+ &freqdomain_cpus,
+#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
+ &cpb,
+#endif
+ NULL,
+};
+
+static struct cpufreq_driver acpi_cpufreq_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = acpi_cpufreq_target,
+ .fast_switch = acpi_cpufreq_fast_switch,
+ .bios_limit = acpi_processor_get_bios_limit,
+ .init = acpi_cpufreq_cpu_init,
+ .exit = acpi_cpufreq_cpu_exit,
+ .resume = acpi_cpufreq_resume,
+ .name = "acpi-cpufreq",
+ .attr = acpi_cpufreq_attr,
+};
+
+static void __init acpi_cpufreq_boost_init(void)
+{
+ if (!(boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA))) {
+ pr_debug("Boost capabilities not present in the processor\n");
+ return;
+ }
+
+ acpi_cpufreq_driver.set_boost = set_boost;
+ acpi_cpufreq_driver.boost_enabled = boost_state(0);
+}
+
+static int __init acpi_cpufreq_probe(struct platform_device *pdev)
+{
+ int ret;
+
+ if (acpi_disabled)
+ return -ENODEV;
+
+ /* don't keep reloading if cpufreq_driver exists */
+ if (cpufreq_get_current_driver())
+ return -ENODEV;
+
+ pr_debug("%s\n", __func__);
+
+ ret = acpi_cpufreq_early_init();
+ if (ret)
+ return ret;
+
+#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
+ /* this is a sysfs file with a strange name and an even stranger
+ * semantic - per CPU instantiation, but system global effect.
+ * Lets enable it only on AMD CPUs for compatibility reasons and
+ * only if configured. This is considered legacy code, which
+ * will probably be removed at some point in the future.
+ */
+ if (!check_amd_hwpstate_cpu(0)) {
+ struct freq_attr **attr;
+
+ pr_debug("CPB unsupported, do not expose it\n");
+
+ for (attr = acpi_cpufreq_attr; *attr; attr++)
+ if (*attr == &cpb) {
+ *attr = NULL;
+ break;
+ }
+ }
+#endif
+ acpi_cpufreq_boost_init();
+
+ ret = cpufreq_register_driver(&acpi_cpufreq_driver);
+ if (ret) {
+ free_acpi_perf_data();
+ }
+ return ret;
+}
+
+static void acpi_cpufreq_remove(struct platform_device *pdev)
+{
+ pr_debug("%s\n", __func__);
+
+ cpufreq_unregister_driver(&acpi_cpufreq_driver);
+
+ free_acpi_perf_data();
+}
+
+static struct platform_driver acpi_cpufreq_platdrv = {
+ .driver = {
+ .name = "acpi-cpufreq",
+ },
+ .remove_new = acpi_cpufreq_remove,
+};
+
+static int __init acpi_cpufreq_init(void)
+{
+ return platform_driver_probe(&acpi_cpufreq_platdrv, acpi_cpufreq_probe);
+}
+
+static void __exit acpi_cpufreq_exit(void)
+{
+ platform_driver_unregister(&acpi_cpufreq_platdrv);
+}
+
+module_param(acpi_pstate_strict, uint, 0644);
+MODULE_PARM_DESC(acpi_pstate_strict,
+ "value 0 or non-zero. non-zero -> strict ACPI checks are "
+ "performed during frequency changes.");
+
+late_initcall(acpi_cpufreq_init);
+module_exit(acpi_cpufreq_exit);
+
+MODULE_ALIAS("platform:acpi-cpufreq");
diff --git a/drivers/cpufreq/amd-pstate-trace.c b/drivers/cpufreq/amd-pstate-trace.c
new file mode 100644
index 0000000000..891b696dcd
--- /dev/null
+++ b/drivers/cpufreq/amd-pstate-trace.c
@@ -0,0 +1,2 @@
+#define CREATE_TRACE_POINTS
+#include "amd-pstate-trace.h"
diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h
new file mode 100644
index 0000000000..35f38ae67f
--- /dev/null
+++ b/drivers/cpufreq/amd-pstate-trace.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * amd-pstate-trace.h - AMD Processor P-state Frequency Driver Tracer
+ *
+ * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ * Author: Huang Rui <ray.huang@amd.com>
+ */
+
+#if !defined(_AMD_PSTATE_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _AMD_PSTATE_TRACE_H
+
+#include <linux/cpufreq.h>
+#include <linux/tracepoint.h>
+#include <linux/trace_events.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM amd_cpu
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE amd-pstate-trace
+
+#define TPS(x) tracepoint_string(x)
+
+TRACE_EVENT(amd_pstate_perf,
+
+ TP_PROTO(unsigned long min_perf,
+ unsigned long target_perf,
+ unsigned long capacity,
+ u64 freq,
+ u64 mperf,
+ u64 aperf,
+ u64 tsc,
+ unsigned int cpu_id,
+ bool changed,
+ bool fast_switch
+ ),
+
+ TP_ARGS(min_perf,
+ target_perf,
+ capacity,
+ freq,
+ mperf,
+ aperf,
+ tsc,
+ cpu_id,
+ changed,
+ fast_switch
+ ),
+
+ TP_STRUCT__entry(
+ __field(unsigned long, min_perf)
+ __field(unsigned long, target_perf)
+ __field(unsigned long, capacity)
+ __field(unsigned long long, freq)
+ __field(unsigned long long, mperf)
+ __field(unsigned long long, aperf)
+ __field(unsigned long long, tsc)
+ __field(unsigned int, cpu_id)
+ __field(bool, changed)
+ __field(bool, fast_switch)
+ ),
+
+ TP_fast_assign(
+ __entry->min_perf = min_perf;
+ __entry->target_perf = target_perf;
+ __entry->capacity = capacity;
+ __entry->freq = freq;
+ __entry->mperf = mperf;
+ __entry->aperf = aperf;
+ __entry->tsc = tsc;
+ __entry->cpu_id = cpu_id;
+ __entry->changed = changed;
+ __entry->fast_switch = fast_switch;
+ ),
+
+ TP_printk("amd_min_perf=%lu amd_des_perf=%lu amd_max_perf=%lu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u changed=%s fast_switch=%s",
+ (unsigned long)__entry->min_perf,
+ (unsigned long)__entry->target_perf,
+ (unsigned long)__entry->capacity,
+ (unsigned long long)__entry->freq,
+ (unsigned long long)__entry->mperf,
+ (unsigned long long)__entry->aperf,
+ (unsigned long long)__entry->tsc,
+ (unsigned int)__entry->cpu_id,
+ (__entry->changed) ? "true" : "false",
+ (__entry->fast_switch) ? "true" : "false"
+ )
+);
+
+#endif /* _AMD_PSTATE_TRACE_H */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#include <trace/define_trace.h>
diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c
new file mode 100644
index 0000000000..f04ae67dda
--- /dev/null
+++ b/drivers/cpufreq/amd-pstate-ut.c
@@ -0,0 +1,283 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * AMD Processor P-state Frequency Driver Unit Test
+ *
+ * Copyright (C) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ * Author: Meng Li <li.meng@amd.com>
+ *
+ * The AMD P-State Unit Test is a test module for testing the amd-pstate
+ * driver. 1) It can help all users to verify their processor support
+ * (SBIOS/Firmware or Hardware). 2) Kernel can have a basic function
+ * test to avoid the kernel regression during the update. 3) We can
+ * introduce more functional or performance tests to align the result
+ * together, it will benefit power and performance scale optimization.
+ *
+ * This driver implements basic framework with plans to enhance it with
+ * additional test cases to improve the depth and coverage of the test.
+ *
+ * See Documentation/admin-guide/pm/amd-pstate.rst Unit Tests for
+ * amd-pstate to get more detail.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/fs.h>
+#include <linux/amd-pstate.h>
+
+#include <acpi/cppc_acpi.h>
+
+/*
+ * Abbreviations:
+ * amd_pstate_ut: used as a shortform for AMD P-State unit test.
+ * It helps to keep variable names smaller, simpler
+ */
+enum amd_pstate_ut_result {
+ AMD_PSTATE_UT_RESULT_PASS,
+ AMD_PSTATE_UT_RESULT_FAIL,
+};
+
+struct amd_pstate_ut_struct {
+ const char *name;
+ void (*func)(u32 index);
+ enum amd_pstate_ut_result result;
+};
+
+/*
+ * Kernel module for testing the AMD P-State unit test
+ */
+static void amd_pstate_ut_acpi_cpc_valid(u32 index);
+static void amd_pstate_ut_check_enabled(u32 index);
+static void amd_pstate_ut_check_perf(u32 index);
+static void amd_pstate_ut_check_freq(u32 index);
+
+static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = {
+ {"amd_pstate_ut_acpi_cpc_valid", amd_pstate_ut_acpi_cpc_valid },
+ {"amd_pstate_ut_check_enabled", amd_pstate_ut_check_enabled },
+ {"amd_pstate_ut_check_perf", amd_pstate_ut_check_perf },
+ {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq }
+};
+
+static bool get_shared_mem(void)
+{
+ bool result = false;
+
+ if (!boot_cpu_has(X86_FEATURE_CPPC))
+ result = true;
+
+ return result;
+}
+
+/*
+ * check the _CPC object is present in SBIOS.
+ */
+static void amd_pstate_ut_acpi_cpc_valid(u32 index)
+{
+ if (acpi_cpc_valid())
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+ else {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s the _CPC object is not present in SBIOS!\n", __func__);
+ }
+}
+
+static void amd_pstate_ut_pstate_enable(u32 index)
+{
+ int ret = 0;
+ u64 cppc_enable = 0;
+
+ ret = rdmsrl_safe(MSR_AMD_CPPC_ENABLE, &cppc_enable);
+ if (ret) {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s rdmsrl_safe MSR_AMD_CPPC_ENABLE ret=%d error!\n", __func__, ret);
+ return;
+ }
+ if (cppc_enable)
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+ else {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s amd pstate must be enabled!\n", __func__);
+ }
+}
+
+/*
+ * check if amd pstate is enabled
+ */
+static void amd_pstate_ut_check_enabled(u32 index)
+{
+ if (get_shared_mem())
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+ else
+ amd_pstate_ut_pstate_enable(index);
+}
+
+/*
+ * check if performance values are reasonable.
+ * highest_perf >= nominal_perf > lowest_nonlinear_perf > lowest_perf > 0
+ */
+static void amd_pstate_ut_check_perf(u32 index)
+{
+ int cpu = 0, ret = 0;
+ u32 highest_perf = 0, nominal_perf = 0, lowest_nonlinear_perf = 0, lowest_perf = 0;
+ u64 cap1 = 0;
+ struct cppc_perf_caps cppc_perf;
+ struct cpufreq_policy *policy = NULL;
+ struct amd_cpudata *cpudata = NULL;
+
+ for_each_possible_cpu(cpu) {
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy)
+ break;
+ cpudata = policy->driver_data;
+
+ if (get_shared_mem()) {
+ ret = cppc_get_perf_caps(cpu, &cppc_perf);
+ if (ret) {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s cppc_get_perf_caps ret=%d error!\n", __func__, ret);
+ goto skip_test;
+ }
+
+ highest_perf = cppc_perf.highest_perf;
+ nominal_perf = cppc_perf.nominal_perf;
+ lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
+ lowest_perf = cppc_perf.lowest_perf;
+ } else {
+ ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1);
+ if (ret) {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s read CPPC_CAP1 ret=%d error!\n", __func__, ret);
+ goto skip_test;
+ }
+
+ highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
+ nominal_perf = AMD_CPPC_NOMINAL_PERF(cap1);
+ lowest_nonlinear_perf = AMD_CPPC_LOWNONLIN_PERF(cap1);
+ lowest_perf = AMD_CPPC_LOWEST_PERF(cap1);
+ }
+
+ if ((highest_perf != READ_ONCE(cpudata->highest_perf)) ||
+ (nominal_perf != READ_ONCE(cpudata->nominal_perf)) ||
+ (lowest_nonlinear_perf != READ_ONCE(cpudata->lowest_nonlinear_perf)) ||
+ (lowest_perf != READ_ONCE(cpudata->lowest_perf))) {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s cpu%d highest=%d %d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n",
+ __func__, cpu, highest_perf, cpudata->highest_perf,
+ nominal_perf, cpudata->nominal_perf,
+ lowest_nonlinear_perf, cpudata->lowest_nonlinear_perf,
+ lowest_perf, cpudata->lowest_perf);
+ goto skip_test;
+ }
+
+ if (!((highest_perf >= nominal_perf) &&
+ (nominal_perf > lowest_nonlinear_perf) &&
+ (lowest_nonlinear_perf > lowest_perf) &&
+ (lowest_perf > 0))) {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s cpu%d highest=%d >= nominal=%d > lowest_nonlinear=%d > lowest=%d > 0, the formula is incorrect!\n",
+ __func__, cpu, highest_perf, nominal_perf,
+ lowest_nonlinear_perf, lowest_perf);
+ goto skip_test;
+ }
+ cpufreq_cpu_put(policy);
+ }
+
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+ return;
+skip_test:
+ cpufreq_cpu_put(policy);
+}
+
+/*
+ * Check if frequency values are reasonable.
+ * max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0
+ * check max freq when set support boost mode.
+ */
+static void amd_pstate_ut_check_freq(u32 index)
+{
+ int cpu = 0;
+ struct cpufreq_policy *policy = NULL;
+ struct amd_cpudata *cpudata = NULL;
+
+ for_each_possible_cpu(cpu) {
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy)
+ break;
+ cpudata = policy->driver_data;
+
+ if (!((cpudata->max_freq >= cpudata->nominal_freq) &&
+ (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) &&
+ (cpudata->lowest_nonlinear_freq > cpudata->min_freq) &&
+ (cpudata->min_freq > 0))) {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n",
+ __func__, cpu, cpudata->max_freq, cpudata->nominal_freq,
+ cpudata->lowest_nonlinear_freq, cpudata->min_freq);
+ goto skip_test;
+ }
+
+ if (cpudata->min_freq != policy->min) {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s cpu%d cpudata_min_freq=%d policy_min=%d, they should be equal!\n",
+ __func__, cpu, cpudata->min_freq, policy->min);
+ goto skip_test;
+ }
+
+ if (cpudata->boost_supported) {
+ if ((policy->max == cpudata->max_freq) ||
+ (policy->max == cpudata->nominal_freq))
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+ else {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n",
+ __func__, cpu, policy->max, cpudata->max_freq,
+ cpudata->nominal_freq);
+ goto skip_test;
+ }
+ } else {
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+ pr_err("%s cpu%d must support boost!\n", __func__, cpu);
+ goto skip_test;
+ }
+ cpufreq_cpu_put(policy);
+ }
+
+ amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+ return;
+skip_test:
+ cpufreq_cpu_put(policy);
+}
+
+static int __init amd_pstate_ut_init(void)
+{
+ u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases);
+
+ for (i = 0; i < arr_size; i++) {
+ amd_pstate_ut_cases[i].func(i);
+ switch (amd_pstate_ut_cases[i].result) {
+ case AMD_PSTATE_UT_RESULT_PASS:
+ pr_info("%-4d %-20s\t success!\n", i+1, amd_pstate_ut_cases[i].name);
+ break;
+ case AMD_PSTATE_UT_RESULT_FAIL:
+ default:
+ pr_info("%-4d %-20s\t fail!\n", i+1, amd_pstate_ut_cases[i].name);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static void __exit amd_pstate_ut_exit(void)
+{
+}
+
+module_init(amd_pstate_ut_init);
+module_exit(amd_pstate_ut_exit);
+
+MODULE_AUTHOR("Meng Li <li.meng@amd.com>");
+MODULE_DESCRIPTION("AMD P-state driver Test module");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c
new file mode 100644
index 0000000000..1791d37fbc
--- /dev/null
+++ b/drivers/cpufreq/amd-pstate.c
@@ -0,0 +1,1573 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * amd-pstate.c - AMD Processor P-state Frequency Driver
+ *
+ * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ * Author: Huang Rui <ray.huang@amd.com>
+ *
+ * AMD P-State introduces a new CPU performance scaling design for AMD
+ * processors using the ACPI Collaborative Performance and Power Control (CPPC)
+ * feature which works with the AMD SMU firmware providing a finer grained
+ * frequency control range. It is to replace the legacy ACPI P-States control,
+ * allows a flexible, low-latency interface for the Linux kernel to directly
+ * communicate the performance hints to hardware.
+ *
+ * AMD P-State is supported on recent AMD Zen base CPU series include some of
+ * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
+ * P-State supported system. And there are two types of hardware implementations
+ * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
+ * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/cpufreq.h>
+#include <linux/compiler.h>
+#include <linux/dmi.h>
+#include <linux/slab.h>
+#include <linux/acpi.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/uaccess.h>
+#include <linux/static_call.h>
+#include <linux/amd-pstate.h>
+
+#include <acpi/processor.h>
+#include <acpi/cppc_acpi.h>
+
+#include <asm/msr.h>
+#include <asm/processor.h>
+#include <asm/cpufeature.h>
+#include <asm/cpu_device_id.h>
+#include "amd-pstate-trace.h"
+
+#define AMD_PSTATE_TRANSITION_LATENCY 20000
+#define AMD_PSTATE_TRANSITION_DELAY 1000
+
+/*
+ * TODO: We need more time to fine tune processors with shared memory solution
+ * with community together.
+ *
+ * There are some performance drops on the CPU benchmarks which reports from
+ * Suse. We are co-working with them to fine tune the shared memory solution. So
+ * we disable it by default to go acpi-cpufreq on these processors and add a
+ * module parameter to be able to enable it manually for debugging.
+ */
+static struct cpufreq_driver *current_pstate_driver;
+static struct cpufreq_driver amd_pstate_driver;
+static struct cpufreq_driver amd_pstate_epp_driver;
+static int cppc_state = AMD_PSTATE_UNDEFINED;
+static bool cppc_enabled;
+
+/*
+ * AMD Energy Preference Performance (EPP)
+ * The EPP is used in the CCLK DPM controller to drive
+ * the frequency that a core is going to operate during
+ * short periods of activity. EPP values will be utilized for
+ * different OS profiles (balanced, performance, power savings)
+ * display strings corresponding to EPP index in the
+ * energy_perf_strings[]
+ * index String
+ *-------------------------------------
+ * 0 default
+ * 1 performance
+ * 2 balance_performance
+ * 3 balance_power
+ * 4 power
+ */
+enum energy_perf_value_index {
+ EPP_INDEX_DEFAULT = 0,
+ EPP_INDEX_PERFORMANCE,
+ EPP_INDEX_BALANCE_PERFORMANCE,
+ EPP_INDEX_BALANCE_POWERSAVE,
+ EPP_INDEX_POWERSAVE,
+};
+
+static const char * const energy_perf_strings[] = {
+ [EPP_INDEX_DEFAULT] = "default",
+ [EPP_INDEX_PERFORMANCE] = "performance",
+ [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
+ [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
+ [EPP_INDEX_POWERSAVE] = "power",
+ NULL
+};
+
+static unsigned int epp_values[] = {
+ [EPP_INDEX_DEFAULT] = 0,
+ [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
+ [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
+ [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
+ [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
+ };
+
+typedef int (*cppc_mode_transition_fn)(int);
+
+static inline int get_mode_idx_from_str(const char *str, size_t size)
+{
+ int i;
+
+ for (i=0; i < AMD_PSTATE_MAX; i++) {
+ if (!strncmp(str, amd_pstate_mode_string[i], size))
+ return i;
+ }
+ return -EINVAL;
+}
+
+static DEFINE_MUTEX(amd_pstate_limits_lock);
+static DEFINE_MUTEX(amd_pstate_driver_lock);
+
+static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)
+{
+ u64 epp;
+ int ret;
+
+ if (boot_cpu_has(X86_FEATURE_CPPC)) {
+ if (!cppc_req_cached) {
+ epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
+ &cppc_req_cached);
+ if (epp)
+ return epp;
+ }
+ epp = (cppc_req_cached >> 24) & 0xFF;
+ } else {
+ ret = cppc_get_epp_perf(cpudata->cpu, &epp);
+ if (ret < 0) {
+ pr_debug("Could not retrieve energy perf value (%d)\n", ret);
+ return -EIO;
+ }
+ }
+
+ return (s16)(epp & 0xff);
+}
+
+static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)
+{
+ s16 epp;
+ int index = -EINVAL;
+
+ epp = amd_pstate_get_epp(cpudata, 0);
+ if (epp < 0)
+ return epp;
+
+ switch (epp) {
+ case AMD_CPPC_EPP_PERFORMANCE:
+ index = EPP_INDEX_PERFORMANCE;
+ break;
+ case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
+ index = EPP_INDEX_BALANCE_PERFORMANCE;
+ break;
+ case AMD_CPPC_EPP_BALANCE_POWERSAVE:
+ index = EPP_INDEX_BALANCE_POWERSAVE;
+ break;
+ case AMD_CPPC_EPP_POWERSAVE:
+ index = EPP_INDEX_POWERSAVE;
+ break;
+ default:
+ break;
+ }
+
+ return index;
+}
+
+static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)
+{
+ int ret;
+ struct cppc_perf_ctrls perf_ctrls;
+
+ if (boot_cpu_has(X86_FEATURE_CPPC)) {
+ u64 value = READ_ONCE(cpudata->cppc_req_cached);
+
+ value &= ~GENMASK_ULL(31, 24);
+ value |= (u64)epp << 24;
+ WRITE_ONCE(cpudata->cppc_req_cached, value);
+
+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
+ if (!ret)
+ cpudata->epp_cached = epp;
+ } else {
+ perf_ctrls.energy_perf = epp;
+ ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
+ if (ret) {
+ pr_debug("failed to set energy perf value (%d)\n", ret);
+ return ret;
+ }
+ cpudata->epp_cached = epp;
+ }
+
+ return ret;
+}
+
+static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,
+ int pref_index)
+{
+ int epp = -EINVAL;
+ int ret;
+
+ if (!pref_index) {
+ pr_debug("EPP pref_index is invalid\n");
+ return -EINVAL;
+ }
+
+ if (epp == -EINVAL)
+ epp = epp_values[pref_index];
+
+ if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
+ pr_debug("EPP cannot be set under performance policy\n");
+ return -EBUSY;
+ }
+
+ ret = amd_pstate_set_epp(cpudata, epp);
+
+ return ret;
+}
+
+static inline int pstate_enable(bool enable)
+{
+ int ret, cpu;
+ unsigned long logical_proc_id_mask = 0;
+
+ if (enable == cppc_enabled)
+ return 0;
+
+ for_each_present_cpu(cpu) {
+ unsigned long logical_id = topology_logical_die_id(cpu);
+
+ if (test_bit(logical_id, &logical_proc_id_mask))
+ continue;
+
+ set_bit(logical_id, &logical_proc_id_mask);
+
+ ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE,
+ enable);
+ if (ret)
+ return ret;
+ }
+
+ cppc_enabled = enable;
+ return 0;
+}
+
+static int cppc_enable(bool enable)
+{
+ int cpu, ret = 0;
+ struct cppc_perf_ctrls perf_ctrls;
+
+ if (enable == cppc_enabled)
+ return 0;
+
+ for_each_present_cpu(cpu) {
+ ret = cppc_set_enable(cpu, enable);
+ if (ret)
+ return ret;
+
+ /* Enable autonomous mode for EPP */
+ if (cppc_state == AMD_PSTATE_ACTIVE) {
+ /* Set desired perf as zero to allow EPP firmware control */
+ perf_ctrls.desired_perf = 0;
+ ret = cppc_set_perf(cpu, &perf_ctrls);
+ if (ret)
+ return ret;
+ }
+ }
+
+ cppc_enabled = enable;
+ return ret;
+}
+
+DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
+
+static inline int amd_pstate_enable(bool enable)
+{
+ return static_call(amd_pstate_enable)(enable);
+}
+
+static int pstate_init_perf(struct amd_cpudata *cpudata)
+{
+ u64 cap1;
+ u32 highest_perf;
+
+ int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
+ &cap1);
+ if (ret)
+ return ret;
+
+ /*
+ * TODO: Introduce AMD specific power feature.
+ *
+ * CPPC entry doesn't indicate the highest performance in some ASICs.
+ */
+ highest_perf = amd_get_highest_perf();
+ if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
+ highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
+
+ WRITE_ONCE(cpudata->highest_perf, highest_perf);
+ WRITE_ONCE(cpudata->max_limit_perf, highest_perf);
+ WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
+ WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
+ WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
+ WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1));
+ return 0;
+}
+
+static int cppc_init_perf(struct amd_cpudata *cpudata)
+{
+ struct cppc_perf_caps cppc_perf;
+ u32 highest_perf;
+
+ int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
+ if (ret)
+ return ret;
+
+ highest_perf = amd_get_highest_perf();
+ if (highest_perf > cppc_perf.highest_perf)
+ highest_perf = cppc_perf.highest_perf;
+
+ WRITE_ONCE(cpudata->highest_perf, highest_perf);
+ WRITE_ONCE(cpudata->max_limit_perf, highest_perf);
+ WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
+ WRITE_ONCE(cpudata->lowest_nonlinear_perf,
+ cppc_perf.lowest_nonlinear_perf);
+ WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
+ WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf);
+
+ if (cppc_state == AMD_PSTATE_ACTIVE)
+ return 0;
+
+ ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf);
+ if (ret) {
+ pr_warn("failed to get auto_sel, ret: %d\n", ret);
+ return 0;
+ }
+
+ ret = cppc_set_auto_sel(cpudata->cpu,
+ (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
+
+ if (ret)
+ pr_warn("failed to set auto_sel, ret: %d\n", ret);
+
+ return ret;
+}
+
+DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
+
+static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
+{
+ return static_call(amd_pstate_init_perf)(cpudata);
+}
+
+static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
+ u32 des_perf, u32 max_perf, bool fast_switch)
+{
+ if (fast_switch)
+ wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
+ else
+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
+ READ_ONCE(cpudata->cppc_req_cached));
+}
+
+static void cppc_update_perf(struct amd_cpudata *cpudata,
+ u32 min_perf, u32 des_perf,
+ u32 max_perf, bool fast_switch)
+{
+ struct cppc_perf_ctrls perf_ctrls;
+
+ perf_ctrls.max_perf = max_perf;
+ perf_ctrls.min_perf = min_perf;
+ perf_ctrls.desired_perf = des_perf;
+
+ cppc_set_perf(cpudata->cpu, &perf_ctrls);
+}
+
+DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
+
+static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
+ u32 min_perf, u32 des_perf,
+ u32 max_perf, bool fast_switch)
+{
+ static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
+ max_perf, fast_switch);
+}
+
+static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
+{
+ u64 aperf, mperf, tsc;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ rdmsrl(MSR_IA32_APERF, aperf);
+ rdmsrl(MSR_IA32_MPERF, mperf);
+ tsc = rdtsc();
+
+ if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
+ local_irq_restore(flags);
+ return false;
+ }
+
+ local_irq_restore(flags);
+
+ cpudata->cur.aperf = aperf;
+ cpudata->cur.mperf = mperf;
+ cpudata->cur.tsc = tsc;
+ cpudata->cur.aperf -= cpudata->prev.aperf;
+ cpudata->cur.mperf -= cpudata->prev.mperf;
+ cpudata->cur.tsc -= cpudata->prev.tsc;
+
+ cpudata->prev.aperf = aperf;
+ cpudata->prev.mperf = mperf;
+ cpudata->prev.tsc = tsc;
+
+ cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
+
+ return true;
+}
+
+static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
+ u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags)
+{
+ u64 prev = READ_ONCE(cpudata->cppc_req_cached);
+ u64 value = prev;
+
+ min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
+ cpudata->max_limit_perf);
+ max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
+ cpudata->max_limit_perf);
+ des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
+
+ if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) {
+ min_perf = des_perf;
+ des_perf = 0;
+ }
+
+ value &= ~AMD_CPPC_MIN_PERF(~0L);
+ value |= AMD_CPPC_MIN_PERF(min_perf);
+
+ value &= ~AMD_CPPC_DES_PERF(~0L);
+ value |= AMD_CPPC_DES_PERF(des_perf);
+
+ value &= ~AMD_CPPC_MAX_PERF(~0L);
+ value |= AMD_CPPC_MAX_PERF(max_perf);
+
+ if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
+ trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
+ cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
+ cpudata->cpu, (value != prev), fast_switch);
+ }
+
+ if (value == prev)
+ return;
+
+ WRITE_ONCE(cpudata->cppc_req_cached, value);
+
+ amd_pstate_update_perf(cpudata, min_perf, des_perf,
+ max_perf, fast_switch);
+}
+
+static int amd_pstate_verify(struct cpufreq_policy_data *policy)
+{
+ cpufreq_verify_within_cpu_limits(policy);
+
+ return 0;
+}
+
+static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy)
+{
+ u32 max_limit_perf, min_limit_perf;
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
+ min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
+
+ WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
+ WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
+ WRITE_ONCE(cpudata->max_limit_freq, policy->max);
+ WRITE_ONCE(cpudata->min_limit_freq, policy->min);
+
+ return 0;
+}
+
+static int amd_pstate_update_freq(struct cpufreq_policy *policy,
+ unsigned int target_freq, bool fast_switch)
+{
+ struct cpufreq_freqs freqs;
+ struct amd_cpudata *cpudata = policy->driver_data;
+ unsigned long max_perf, min_perf, des_perf, cap_perf;
+
+ if (!cpudata->max_freq)
+ return -ENODEV;
+
+ if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
+ amd_pstate_update_min_max_limit(policy);
+
+ cap_perf = READ_ONCE(cpudata->highest_perf);
+ min_perf = READ_ONCE(cpudata->lowest_perf);
+ max_perf = cap_perf;
+
+ freqs.old = policy->cur;
+ freqs.new = target_freq;
+
+ des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
+ cpudata->max_freq);
+
+ WARN_ON(fast_switch && !policy->fast_switch_enabled);
+ /*
+ * If fast_switch is desired, then there aren't any registered
+ * transition notifiers. See comment for
+ * cpufreq_enable_fast_switch().
+ */
+ if (!fast_switch)
+ cpufreq_freq_transition_begin(policy, &freqs);
+
+ amd_pstate_update(cpudata, min_perf, des_perf,
+ max_perf, fast_switch, policy->governor->flags);
+
+ if (!fast_switch)
+ cpufreq_freq_transition_end(policy, &freqs, false);
+
+ return 0;
+}
+
+static int amd_pstate_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ return amd_pstate_update_freq(policy, target_freq, false);
+}
+
+static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ if (!amd_pstate_update_freq(policy, target_freq, true))
+ return target_freq;
+ return policy->cur;
+}
+
+static void amd_pstate_adjust_perf(unsigned int cpu,
+ unsigned long _min_perf,
+ unsigned long target_perf,
+ unsigned long capacity)
+{
+ unsigned long max_perf, min_perf, des_perf,
+ cap_perf, lowest_nonlinear_perf, max_freq;
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+ struct amd_cpudata *cpudata = policy->driver_data;
+ unsigned int target_freq;
+
+ if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
+ amd_pstate_update_min_max_limit(policy);
+
+
+ cap_perf = READ_ONCE(cpudata->highest_perf);
+ lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
+ max_freq = READ_ONCE(cpudata->max_freq);
+
+ des_perf = cap_perf;
+ if (target_perf < capacity)
+ des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
+
+ min_perf = READ_ONCE(cpudata->highest_perf);
+ if (_min_perf < capacity)
+ min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
+
+ if (min_perf < lowest_nonlinear_perf)
+ min_perf = lowest_nonlinear_perf;
+
+ max_perf = cap_perf;
+ if (max_perf < min_perf)
+ max_perf = min_perf;
+
+ des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
+ target_freq = div_u64(des_perf * max_freq, max_perf);
+ policy->cur = target_freq;
+
+ amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true,
+ policy->governor->flags);
+ cpufreq_cpu_put(policy);
+}
+
+static int amd_get_min_freq(struct amd_cpudata *cpudata)
+{
+ struct cppc_perf_caps cppc_perf;
+
+ int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
+ if (ret)
+ return ret;
+
+ /* Switch to khz */
+ return cppc_perf.lowest_freq * 1000;
+}
+
+static int amd_get_max_freq(struct amd_cpudata *cpudata)
+{
+ struct cppc_perf_caps cppc_perf;
+ u32 max_perf, max_freq, nominal_freq, nominal_perf;
+ u64 boost_ratio;
+
+ int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
+ if (ret)
+ return ret;
+
+ nominal_freq = cppc_perf.nominal_freq;
+ nominal_perf = READ_ONCE(cpudata->nominal_perf);
+ max_perf = READ_ONCE(cpudata->highest_perf);
+
+ boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
+ nominal_perf);
+
+ max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
+
+ /* Switch to khz */
+ return max_freq * 1000;
+}
+
+static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
+{
+ struct cppc_perf_caps cppc_perf;
+
+ int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
+ if (ret)
+ return ret;
+
+ /* Switch to khz */
+ return cppc_perf.nominal_freq * 1000;
+}
+
+static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
+{
+ struct cppc_perf_caps cppc_perf;
+ u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
+ nominal_freq, nominal_perf;
+ u64 lowest_nonlinear_ratio;
+
+ int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
+ if (ret)
+ return ret;
+
+ nominal_freq = cppc_perf.nominal_freq;
+ nominal_perf = READ_ONCE(cpudata->nominal_perf);
+
+ lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
+
+ lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
+ nominal_perf);
+
+ lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
+
+ /* Switch to khz */
+ return lowest_nonlinear_freq * 1000;
+}
+
+static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+ int ret;
+
+ if (!cpudata->boost_supported) {
+ pr_err("Boost mode is not supported by this processor or SBIOS\n");
+ return -EINVAL;
+ }
+
+ if (state)
+ policy->cpuinfo.max_freq = cpudata->max_freq;
+ else
+ policy->cpuinfo.max_freq = cpudata->nominal_freq;
+
+ policy->max = policy->cpuinfo.max_freq;
+
+ ret = freq_qos_update_request(&cpudata->req[1],
+ policy->cpuinfo.max_freq);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
+{
+ u32 highest_perf, nominal_perf;
+
+ highest_perf = READ_ONCE(cpudata->highest_perf);
+ nominal_perf = READ_ONCE(cpudata->nominal_perf);
+
+ if (highest_perf <= nominal_perf)
+ return;
+
+ cpudata->boost_supported = true;
+ current_pstate_driver->boost_enabled = true;
+}
+
+static void amd_perf_ctl_reset(unsigned int cpu)
+{
+ wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
+}
+
+static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
+{
+ int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
+ struct device *dev;
+ struct amd_cpudata *cpudata;
+
+ /*
+ * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
+ * which is ideal for initialization process.
+ */
+ amd_perf_ctl_reset(policy->cpu);
+ dev = get_cpu_device(policy->cpu);
+ if (!dev)
+ return -ENODEV;
+
+ cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
+ if (!cpudata)
+ return -ENOMEM;
+
+ cpudata->cpu = policy->cpu;
+
+ ret = amd_pstate_init_perf(cpudata);
+ if (ret)
+ goto free_cpudata1;
+
+ min_freq = amd_get_min_freq(cpudata);
+ max_freq = amd_get_max_freq(cpudata);
+ nominal_freq = amd_get_nominal_freq(cpudata);
+ lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
+
+ if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
+ dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
+ min_freq, max_freq);
+ ret = -EINVAL;
+ goto free_cpudata1;
+ }
+
+ policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
+ policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
+
+ policy->min = min_freq;
+ policy->max = max_freq;
+
+ policy->cpuinfo.min_freq = min_freq;
+ policy->cpuinfo.max_freq = max_freq;
+
+ /* It will be updated by governor */
+ policy->cur = policy->cpuinfo.min_freq;
+
+ if (boot_cpu_has(X86_FEATURE_CPPC))
+ policy->fast_switch_possible = true;
+
+ ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
+ FREQ_QOS_MIN, policy->cpuinfo.min_freq);
+ if (ret < 0) {
+ dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
+ goto free_cpudata1;
+ }
+
+ ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
+ FREQ_QOS_MAX, policy->cpuinfo.max_freq);
+ if (ret < 0) {
+ dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
+ goto free_cpudata2;
+ }
+
+ /* Initial processor data capability frequencies */
+ cpudata->max_freq = max_freq;
+ cpudata->min_freq = min_freq;
+ cpudata->max_limit_freq = max_freq;
+ cpudata->min_limit_freq = min_freq;
+ cpudata->nominal_freq = nominal_freq;
+ cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
+
+ policy->driver_data = cpudata;
+
+ amd_pstate_boost_init(cpudata);
+ if (!current_pstate_driver->adjust_perf)
+ current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
+
+ return 0;
+
+free_cpudata2:
+ freq_qos_remove_request(&cpudata->req[0]);
+free_cpudata1:
+ kfree(cpudata);
+ return ret;
+}
+
+static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ freq_qos_remove_request(&cpudata->req[1]);
+ freq_qos_remove_request(&cpudata->req[0]);
+ policy->fast_switch_possible = false;
+ kfree(cpudata);
+
+ return 0;
+}
+
+static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
+{
+ int ret;
+
+ ret = amd_pstate_enable(true);
+ if (ret)
+ pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
+
+ return ret;
+}
+
+static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
+{
+ int ret;
+
+ ret = amd_pstate_enable(false);
+ if (ret)
+ pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
+
+ return ret;
+}
+
+/* Sysfs attributes */
+
+/*
+ * This frequency is to indicate the maximum hardware frequency.
+ * If boost is not active but supported, the frequency will be larger than the
+ * one in cpuinfo.
+ */
+static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
+ char *buf)
+{
+ int max_freq;
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ max_freq = amd_get_max_freq(cpudata);
+ if (max_freq < 0)
+ return max_freq;
+
+ return sysfs_emit(buf, "%u\n", max_freq);
+}
+
+static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
+ char *buf)
+{
+ int freq;
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ freq = amd_get_lowest_nonlinear_freq(cpudata);
+ if (freq < 0)
+ return freq;
+
+ return sysfs_emit(buf, "%u\n", freq);
+}
+
+/*
+ * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
+ * need to expose it to sysfs.
+ */
+static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
+ char *buf)
+{
+ u32 perf;
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ perf = READ_ONCE(cpudata->highest_perf);
+
+ return sysfs_emit(buf, "%u\n", perf);
+}
+
+static ssize_t show_energy_performance_available_preferences(
+ struct cpufreq_policy *policy, char *buf)
+{
+ int i = 0;
+ int offset = 0;
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
+ return sysfs_emit_at(buf, offset, "%s\n",
+ energy_perf_strings[EPP_INDEX_PERFORMANCE]);
+
+ while (energy_perf_strings[i] != NULL)
+ offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]);
+
+ offset += sysfs_emit_at(buf, offset, "\n");
+
+ return offset;
+}
+
+static ssize_t store_energy_performance_preference(
+ struct cpufreq_policy *policy, const char *buf, size_t count)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+ char str_preference[21];
+ ssize_t ret;
+
+ ret = sscanf(buf, "%20s", str_preference);
+ if (ret != 1)
+ return -EINVAL;
+
+ ret = match_string(energy_perf_strings, -1, str_preference);
+ if (ret < 0)
+ return -EINVAL;
+
+ mutex_lock(&amd_pstate_limits_lock);
+ ret = amd_pstate_set_energy_pref_index(cpudata, ret);
+ mutex_unlock(&amd_pstate_limits_lock);
+
+ return ret ?: count;
+}
+
+static ssize_t show_energy_performance_preference(
+ struct cpufreq_policy *policy, char *buf)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+ int preference;
+
+ preference = amd_pstate_get_energy_pref_index(cpudata);
+ if (preference < 0)
+ return preference;
+
+ return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
+}
+
+static void amd_pstate_driver_cleanup(void)
+{
+ amd_pstate_enable(false);
+ cppc_state = AMD_PSTATE_DISABLE;
+ current_pstate_driver = NULL;
+}
+
+static int amd_pstate_register_driver(int mode)
+{
+ int ret;
+
+ if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED)
+ current_pstate_driver = &amd_pstate_driver;
+ else if (mode == AMD_PSTATE_ACTIVE)
+ current_pstate_driver = &amd_pstate_epp_driver;
+ else
+ return -EINVAL;
+
+ cppc_state = mode;
+ ret = cpufreq_register_driver(current_pstate_driver);
+ if (ret) {
+ amd_pstate_driver_cleanup();
+ return ret;
+ }
+ return 0;
+}
+
+static int amd_pstate_unregister_driver(int dummy)
+{
+ cpufreq_unregister_driver(current_pstate_driver);
+ amd_pstate_driver_cleanup();
+ return 0;
+}
+
+static int amd_pstate_change_mode_without_dvr_change(int mode)
+{
+ int cpu = 0;
+
+ cppc_state = mode;
+
+ if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE)
+ return 0;
+
+ for_each_present_cpu(cpu) {
+ cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
+ }
+
+ return 0;
+}
+
+static int amd_pstate_change_driver_mode(int mode)
+{
+ int ret;
+
+ ret = amd_pstate_unregister_driver(0);
+ if (ret)
+ return ret;
+
+ ret = amd_pstate_register_driver(mode);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = {
+ [AMD_PSTATE_DISABLE] = {
+ [AMD_PSTATE_DISABLE] = NULL,
+ [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver,
+ [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver,
+ [AMD_PSTATE_GUIDED] = amd_pstate_register_driver,
+ },
+ [AMD_PSTATE_PASSIVE] = {
+ [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
+ [AMD_PSTATE_PASSIVE] = NULL,
+ [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
+ [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change,
+ },
+ [AMD_PSTATE_ACTIVE] = {
+ [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
+ [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode,
+ [AMD_PSTATE_ACTIVE] = NULL,
+ [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode,
+ },
+ [AMD_PSTATE_GUIDED] = {
+ [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
+ [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change,
+ [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
+ [AMD_PSTATE_GUIDED] = NULL,
+ },
+};
+
+static ssize_t amd_pstate_show_status(char *buf)
+{
+ if (!current_pstate_driver)
+ return sysfs_emit(buf, "disable\n");
+
+ return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
+}
+
+static int amd_pstate_update_status(const char *buf, size_t size)
+{
+ int mode_idx;
+
+ if (size > strlen("passive") || size < strlen("active"))
+ return -EINVAL;
+
+ mode_idx = get_mode_idx_from_str(buf, size);
+
+ if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX)
+ return -EINVAL;
+
+ if (mode_state_machine[cppc_state][mode_idx])
+ return mode_state_machine[cppc_state][mode_idx](mode_idx);
+
+ return 0;
+}
+
+static ssize_t status_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ ssize_t ret;
+
+ mutex_lock(&amd_pstate_driver_lock);
+ ret = amd_pstate_show_status(buf);
+ mutex_unlock(&amd_pstate_driver_lock);
+
+ return ret;
+}
+
+static ssize_t status_store(struct device *a, struct device_attribute *b,
+ const char *buf, size_t count)
+{
+ char *p = memchr(buf, '\n', count);
+ int ret;
+
+ mutex_lock(&amd_pstate_driver_lock);
+ ret = amd_pstate_update_status(buf, p ? p - buf : count);
+ mutex_unlock(&amd_pstate_driver_lock);
+
+ return ret < 0 ? ret : count;
+}
+
+cpufreq_freq_attr_ro(amd_pstate_max_freq);
+cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
+
+cpufreq_freq_attr_ro(amd_pstate_highest_perf);
+cpufreq_freq_attr_rw(energy_performance_preference);
+cpufreq_freq_attr_ro(energy_performance_available_preferences);
+static DEVICE_ATTR_RW(status);
+
+static struct freq_attr *amd_pstate_attr[] = {
+ &amd_pstate_max_freq,
+ &amd_pstate_lowest_nonlinear_freq,
+ &amd_pstate_highest_perf,
+ NULL,
+};
+
+static struct freq_attr *amd_pstate_epp_attr[] = {
+ &amd_pstate_max_freq,
+ &amd_pstate_lowest_nonlinear_freq,
+ &amd_pstate_highest_perf,
+ &energy_performance_preference,
+ &energy_performance_available_preferences,
+ NULL,
+};
+
+static struct attribute *pstate_global_attributes[] = {
+ &dev_attr_status.attr,
+ NULL
+};
+
+static const struct attribute_group amd_pstate_global_attr_group = {
+ .name = "amd_pstate",
+ .attrs = pstate_global_attributes,
+};
+
+static bool amd_pstate_acpi_pm_profile_server(void)
+{
+ switch (acpi_gbl_FADT.preferred_profile) {
+ case PM_ENTERPRISE_SERVER:
+ case PM_SOHO_SERVER:
+ case PM_PERFORMANCE_SERVER:
+ return true;
+ }
+ return false;
+}
+
+static bool amd_pstate_acpi_pm_profile_undefined(void)
+{
+ if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED)
+ return true;
+ if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES)
+ return true;
+ return false;
+}
+
+static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
+{
+ int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
+ struct amd_cpudata *cpudata;
+ struct device *dev;
+ u64 value;
+
+ /*
+ * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
+ * which is ideal for initialization process.
+ */
+ amd_perf_ctl_reset(policy->cpu);
+ dev = get_cpu_device(policy->cpu);
+ if (!dev)
+ return -ENODEV;
+
+ cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
+ if (!cpudata)
+ return -ENOMEM;
+
+ cpudata->cpu = policy->cpu;
+ cpudata->epp_policy = 0;
+
+ ret = amd_pstate_init_perf(cpudata);
+ if (ret)
+ goto free_cpudata1;
+
+ min_freq = amd_get_min_freq(cpudata);
+ max_freq = amd_get_max_freq(cpudata);
+ nominal_freq = amd_get_nominal_freq(cpudata);
+ lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
+ if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
+ dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
+ min_freq, max_freq);
+ ret = -EINVAL;
+ goto free_cpudata1;
+ }
+
+ policy->cpuinfo.min_freq = min_freq;
+ policy->cpuinfo.max_freq = max_freq;
+ /* It will be updated by governor */
+ policy->cur = policy->cpuinfo.min_freq;
+
+ /* Initial processor data capability frequencies */
+ cpudata->max_freq = max_freq;
+ cpudata->min_freq = min_freq;
+ cpudata->nominal_freq = nominal_freq;
+ cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
+
+ policy->driver_data = cpudata;
+
+ cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0);
+
+ policy->min = policy->cpuinfo.min_freq;
+ policy->max = policy->cpuinfo.max_freq;
+
+ /*
+ * Set the policy to provide a valid fallback value in case
+ * the default cpufreq governor is neither powersave nor performance.
+ */
+ if (amd_pstate_acpi_pm_profile_server() ||
+ amd_pstate_acpi_pm_profile_undefined())
+ policy->policy = CPUFREQ_POLICY_PERFORMANCE;
+ else
+ policy->policy = CPUFREQ_POLICY_POWERSAVE;
+
+ if (boot_cpu_has(X86_FEATURE_CPPC)) {
+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
+ if (ret)
+ return ret;
+ WRITE_ONCE(cpudata->cppc_req_cached, value);
+
+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);
+ if (ret)
+ return ret;
+ WRITE_ONCE(cpudata->cppc_cap1_cached, value);
+ }
+ amd_pstate_boost_init(cpudata);
+
+ return 0;
+
+free_cpudata1:
+ kfree(cpudata);
+ return ret;
+}
+
+static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
+{
+ pr_debug("CPU %d exiting\n", policy->cpu);
+ return 0;
+}
+
+static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+ u32 max_perf, min_perf, min_limit_perf, max_limit_perf;
+ u64 value;
+ s16 epp;
+
+ max_perf = READ_ONCE(cpudata->highest_perf);
+ min_perf = READ_ONCE(cpudata->lowest_perf);
+ max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
+ min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
+
+ WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
+ WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
+
+ max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
+ cpudata->max_limit_perf);
+ min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
+ cpudata->max_limit_perf);
+ value = READ_ONCE(cpudata->cppc_req_cached);
+
+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
+ min_perf = max_perf;
+
+ /* Initial min/max values for CPPC Performance Controls Register */
+ value &= ~AMD_CPPC_MIN_PERF(~0L);
+ value |= AMD_CPPC_MIN_PERF(min_perf);
+
+ value &= ~AMD_CPPC_MAX_PERF(~0L);
+ value |= AMD_CPPC_MAX_PERF(max_perf);
+
+ /* CPPC EPP feature require to set zero to the desire perf bit */
+ value &= ~AMD_CPPC_DES_PERF(~0L);
+ value |= AMD_CPPC_DES_PERF(0);
+
+ cpudata->epp_policy = cpudata->policy;
+
+ /* Get BIOS pre-defined epp value */
+ epp = amd_pstate_get_epp(cpudata, value);
+ if (epp < 0) {
+ /**
+ * This return value can only be negative for shared_memory
+ * systems where EPP register read/write not supported.
+ */
+ return;
+ }
+
+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
+ epp = 0;
+
+ /* Set initial EPP value */
+ if (boot_cpu_has(X86_FEATURE_CPPC)) {
+ value &= ~GENMASK_ULL(31, 24);
+ value |= (u64)epp << 24;
+ }
+
+ WRITE_ONCE(cpudata->cppc_req_cached, value);
+ amd_pstate_set_epp(cpudata, epp);
+}
+
+static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ if (!policy->cpuinfo.max_freq)
+ return -ENODEV;
+
+ pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
+ policy->cpuinfo.max_freq, policy->max);
+
+ cpudata->policy = policy->policy;
+
+ amd_pstate_epp_update_limit(policy);
+
+ return 0;
+}
+
+static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)
+{
+ struct cppc_perf_ctrls perf_ctrls;
+ u64 value, max_perf;
+ int ret;
+
+ ret = amd_pstate_enable(true);
+ if (ret)
+ pr_err("failed to enable amd pstate during resume, return %d\n", ret);
+
+ value = READ_ONCE(cpudata->cppc_req_cached);
+ max_perf = READ_ONCE(cpudata->highest_perf);
+
+ if (boot_cpu_has(X86_FEATURE_CPPC)) {
+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
+ } else {
+ perf_ctrls.max_perf = max_perf;
+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);
+ cppc_set_perf(cpudata->cpu, &perf_ctrls);
+ }
+}
+
+static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);
+
+ if (cppc_state == AMD_PSTATE_ACTIVE) {
+ amd_pstate_epp_reenable(cpudata);
+ cpudata->suspended = false;
+ }
+
+ return 0;
+}
+
+static void amd_pstate_epp_offline(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+ struct cppc_perf_ctrls perf_ctrls;
+ int min_perf;
+ u64 value;
+
+ min_perf = READ_ONCE(cpudata->lowest_perf);
+ value = READ_ONCE(cpudata->cppc_req_cached);
+
+ mutex_lock(&amd_pstate_limits_lock);
+ if (boot_cpu_has(X86_FEATURE_CPPC)) {
+ cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;
+
+ /* Set max perf same as min perf */
+ value &= ~AMD_CPPC_MAX_PERF(~0L);
+ value |= AMD_CPPC_MAX_PERF(min_perf);
+ value &= ~AMD_CPPC_MIN_PERF(~0L);
+ value |= AMD_CPPC_MIN_PERF(min_perf);
+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
+ } else {
+ perf_ctrls.desired_perf = 0;
+ perf_ctrls.max_perf = min_perf;
+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE);
+ cppc_set_perf(cpudata->cpu, &perf_ctrls);
+ }
+ mutex_unlock(&amd_pstate_limits_lock);
+}
+
+static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);
+
+ if (cpudata->suspended)
+ return 0;
+
+ if (cppc_state == AMD_PSTATE_ACTIVE)
+ amd_pstate_epp_offline(policy);
+
+ return 0;
+}
+
+static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy)
+{
+ cpufreq_verify_within_cpu_limits(policy);
+ pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min);
+ return 0;
+}
+
+static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+ int ret;
+
+ /* avoid suspending when EPP is not enabled */
+ if (cppc_state != AMD_PSTATE_ACTIVE)
+ return 0;
+
+ /* set this flag to avoid setting core offline*/
+ cpudata->suspended = true;
+
+ /* disable CPPC in lowlevel firmware */
+ ret = amd_pstate_enable(false);
+ if (ret)
+ pr_err("failed to suspend, return %d\n", ret);
+
+ return 0;
+}
+
+static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
+{
+ struct amd_cpudata *cpudata = policy->driver_data;
+
+ if (cpudata->suspended) {
+ mutex_lock(&amd_pstate_limits_lock);
+
+ /* enable amd pstate from suspend state*/
+ amd_pstate_epp_reenable(cpudata);
+
+ mutex_unlock(&amd_pstate_limits_lock);
+
+ cpudata->suspended = false;
+ }
+
+ return 0;
+}
+
+static struct cpufreq_driver amd_pstate_driver = {
+ .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
+ .verify = amd_pstate_verify,
+ .target = amd_pstate_target,
+ .fast_switch = amd_pstate_fast_switch,
+ .init = amd_pstate_cpu_init,
+ .exit = amd_pstate_cpu_exit,
+ .suspend = amd_pstate_cpu_suspend,
+ .resume = amd_pstate_cpu_resume,
+ .set_boost = amd_pstate_set_boost,
+ .name = "amd-pstate",
+ .attr = amd_pstate_attr,
+};
+
+static struct cpufreq_driver amd_pstate_epp_driver = {
+ .flags = CPUFREQ_CONST_LOOPS,
+ .verify = amd_pstate_epp_verify_policy,
+ .setpolicy = amd_pstate_epp_set_policy,
+ .init = amd_pstate_epp_cpu_init,
+ .exit = amd_pstate_epp_cpu_exit,
+ .offline = amd_pstate_epp_cpu_offline,
+ .online = amd_pstate_epp_cpu_online,
+ .suspend = amd_pstate_epp_suspend,
+ .resume = amd_pstate_epp_resume,
+ .name = "amd-pstate-epp",
+ .attr = amd_pstate_epp_attr,
+};
+
+static int __init amd_pstate_set_driver(int mode_idx)
+{
+ if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
+ cppc_state = mode_idx;
+ if (cppc_state == AMD_PSTATE_DISABLE)
+ pr_info("driver is explicitly disabled\n");
+
+ if (cppc_state == AMD_PSTATE_ACTIVE)
+ current_pstate_driver = &amd_pstate_epp_driver;
+
+ if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED)
+ current_pstate_driver = &amd_pstate_driver;
+
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int __init amd_pstate_init(void)
+{
+ struct device *dev_root;
+ int ret;
+
+ if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
+ return -ENODEV;
+
+ if (!acpi_cpc_valid()) {
+ pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
+ return -ENODEV;
+ }
+
+ /* don't keep reloading if cpufreq_driver exists */
+ if (cpufreq_get_current_driver())
+ return -EEXIST;
+
+ switch (cppc_state) {
+ case AMD_PSTATE_UNDEFINED:
+ /* Disable on the following configs by default:
+ * 1. Undefined platforms
+ * 2. Server platforms
+ * 3. Shared memory designs
+ */
+ if (amd_pstate_acpi_pm_profile_undefined() ||
+ amd_pstate_acpi_pm_profile_server() ||
+ !boot_cpu_has(X86_FEATURE_CPPC)) {
+ pr_info("driver load is disabled, boot with specific mode to enable this\n");
+ return -ENODEV;
+ }
+ ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE);
+ if (ret)
+ return ret;
+ break;
+ case AMD_PSTATE_DISABLE:
+ return -ENODEV;
+ case AMD_PSTATE_PASSIVE:
+ case AMD_PSTATE_ACTIVE:
+ case AMD_PSTATE_GUIDED:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ /* capability check */
+ if (boot_cpu_has(X86_FEATURE_CPPC)) {
+ pr_debug("AMD CPPC MSR based functionality is supported\n");
+ if (cppc_state != AMD_PSTATE_ACTIVE)
+ current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
+ } else {
+ pr_debug("AMD CPPC shared memory based functionality is supported\n");
+ static_call_update(amd_pstate_enable, cppc_enable);
+ static_call_update(amd_pstate_init_perf, cppc_init_perf);
+ static_call_update(amd_pstate_update_perf, cppc_update_perf);
+ }
+
+ /* enable amd pstate feature */
+ ret = amd_pstate_enable(true);
+ if (ret) {
+ pr_err("failed to enable with return %d\n", ret);
+ return ret;
+ }
+
+ ret = cpufreq_register_driver(current_pstate_driver);
+ if (ret)
+ pr_err("failed to register with return %d\n", ret);
+
+ dev_root = bus_get_dev_root(&cpu_subsys);
+ if (dev_root) {
+ ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group);
+ put_device(dev_root);
+ if (ret) {
+ pr_err("sysfs attribute export failed with error %d.\n", ret);
+ goto global_attr_free;
+ }
+ }
+
+ return ret;
+
+global_attr_free:
+ cpufreq_unregister_driver(current_pstate_driver);
+ return ret;
+}
+device_initcall(amd_pstate_init);
+
+static int __init amd_pstate_param(char *str)
+{
+ size_t size;
+ int mode_idx;
+
+ if (!str)
+ return -EINVAL;
+
+ size = strlen(str);
+ mode_idx = get_mode_idx_from_str(str, size);
+
+ return amd_pstate_set_driver(mode_idx);
+}
+early_param("amd_pstate", amd_pstate_param);
+
+MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
+MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");
diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c
new file mode 100644
index 0000000000..59b19b9975
--- /dev/null
+++ b/drivers/cpufreq/amd_freq_sensitivity.c
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * amd_freq_sensitivity.c: AMD frequency sensitivity feedback powersave bias
+ * for the ondemand governor.
+ *
+ * Copyright (C) 2013 Advanced Micro Devices, Inc.
+ *
+ * Author: Jacob Shin <jacob.shin@amd.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/percpu-defs.h>
+#include <linux/init.h>
+#include <linux/mod_devicetable.h>
+
+#include <asm/msr.h>
+#include <asm/cpufeature.h>
+#include <asm/cpu_device_id.h>
+
+#include "cpufreq_ondemand.h"
+
+#define MSR_AMD64_FREQ_SENSITIVITY_ACTUAL 0xc0010080
+#define MSR_AMD64_FREQ_SENSITIVITY_REFERENCE 0xc0010081
+#define CLASS_CODE_SHIFT 56
+#define POWERSAVE_BIAS_MAX 1000
+#define POWERSAVE_BIAS_DEF 400
+
+struct cpu_data_t {
+ u64 actual;
+ u64 reference;
+ unsigned int freq_prev;
+};
+
+static DEFINE_PER_CPU(struct cpu_data_t, cpu_data);
+
+static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy,
+ unsigned int freq_next,
+ unsigned int relation)
+{
+ int sensitivity;
+ long d_actual, d_reference;
+ struct msr actual, reference;
+ struct cpu_data_t *data = &per_cpu(cpu_data, policy->cpu);
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct dbs_data *od_data = policy_dbs->dbs_data;
+ struct od_dbs_tuners *od_tuners = od_data->tuners;
+
+ if (!policy->freq_table)
+ return freq_next;
+
+ rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL,
+ &actual.l, &actual.h);
+ rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_REFERENCE,
+ &reference.l, &reference.h);
+ actual.h &= 0x00ffffff;
+ reference.h &= 0x00ffffff;
+
+ /* counter wrapped around, so stay on current frequency */
+ if (actual.q < data->actual || reference.q < data->reference) {
+ freq_next = policy->cur;
+ goto out;
+ }
+
+ d_actual = actual.q - data->actual;
+ d_reference = reference.q - data->reference;
+
+ /* divide by 0, so stay on current frequency as well */
+ if (d_reference == 0) {
+ freq_next = policy->cur;
+ goto out;
+ }
+
+ sensitivity = POWERSAVE_BIAS_MAX -
+ (POWERSAVE_BIAS_MAX * (d_reference - d_actual) / d_reference);
+
+ clamp(sensitivity, 0, POWERSAVE_BIAS_MAX);
+
+ /* this workload is not CPU bound, so choose a lower freq */
+ if (sensitivity < od_tuners->powersave_bias) {
+ if (data->freq_prev == policy->cur)
+ freq_next = policy->cur;
+
+ if (freq_next > policy->cur)
+ freq_next = policy->cur;
+ else if (freq_next < policy->cur)
+ freq_next = policy->min;
+ else {
+ unsigned int index;
+
+ index = cpufreq_table_find_index_h(policy,
+ policy->cur - 1,
+ relation & CPUFREQ_RELATION_E);
+ freq_next = policy->freq_table[index].frequency;
+ }
+
+ data->freq_prev = freq_next;
+ } else
+ data->freq_prev = 0;
+
+out:
+ data->actual = actual.q;
+ data->reference = reference.q;
+ return freq_next;
+}
+
+static int __init amd_freq_sensitivity_init(void)
+{
+ u64 val;
+ struct pci_dev *pcidev;
+ unsigned int pci_vendor;
+
+ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
+ pci_vendor = PCI_VENDOR_ID_AMD;
+ else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
+ pci_vendor = PCI_VENDOR_ID_HYGON;
+ else
+ return -ENODEV;
+
+ pcidev = pci_get_device(pci_vendor,
+ PCI_DEVICE_ID_AMD_KERNCZ_SMBUS, NULL);
+
+ if (!pcidev) {
+ if (!boot_cpu_has(X86_FEATURE_PROC_FEEDBACK))
+ return -ENODEV;
+ } else {
+ pci_dev_put(pcidev);
+ }
+
+ if (rdmsrl_safe(MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, &val))
+ return -ENODEV;
+
+ if (!(val >> CLASS_CODE_SHIFT))
+ return -ENODEV;
+
+ od_register_powersave_bias_handler(amd_powersave_bias_target,
+ POWERSAVE_BIAS_DEF);
+ return 0;
+}
+late_initcall(amd_freq_sensitivity_init);
+
+static void __exit amd_freq_sensitivity_exit(void)
+{
+ od_unregister_powersave_bias_handler();
+}
+module_exit(amd_freq_sensitivity_exit);
+
+static const struct x86_cpu_id __maybe_unused amd_freq_sensitivity_ids[] = {
+ X86_MATCH_FEATURE(X86_FEATURE_PROC_FEEDBACK, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, amd_freq_sensitivity_ids);
+
+MODULE_AUTHOR("Jacob Shin <jacob.shin@amd.com>");
+MODULE_DESCRIPTION("AMD frequency sensitivity feedback powersave bias for "
+ "the ondemand governor.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c
new file mode 100644
index 0000000000..021f423705
--- /dev/null
+++ b/drivers/cpufreq/apple-soc-cpufreq.c
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Apple SoC CPU cluster performance state driver
+ *
+ * Copyright The Asahi Linux Contributors
+ *
+ * Based on scpi-cpufreq.c
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+
+#define APPLE_DVFS_CMD 0x20
+#define APPLE_DVFS_CMD_BUSY BIT(31)
+#define APPLE_DVFS_CMD_SET BIT(25)
+#define APPLE_DVFS_CMD_PS2 GENMASK(16, 12)
+#define APPLE_DVFS_CMD_PS1 GENMASK(4, 0)
+
+/* Same timebase as CPU counter (24MHz) */
+#define APPLE_DVFS_LAST_CHG_TIME 0x38
+
+/*
+ * Apple ran out of bits and had to shift this in T8112...
+ */
+#define APPLE_DVFS_STATUS 0x50
+#define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4)
+#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4
+#define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0)
+#define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5)
+#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5
+#define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0)
+
+/*
+ * Div is +1, base clock is 12MHz on existing SoCs.
+ * For documentation purposes. We use the OPP table to
+ * get the frequency.
+ */
+#define APPLE_DVFS_PLL_STATUS 0xc0
+#define APPLE_DVFS_PLL_FACTOR 0xc8
+#define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16)
+#define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0)
+
+#define APPLE_DVFS_TRANSITION_TIMEOUT 100
+
+struct apple_soc_cpufreq_info {
+ u64 max_pstate;
+ u64 cur_pstate_mask;
+ u64 cur_pstate_shift;
+};
+
+struct apple_cpu_priv {
+ struct device *cpu_dev;
+ void __iomem *reg_base;
+ const struct apple_soc_cpufreq_info *info;
+};
+
+static struct cpufreq_driver apple_soc_cpufreq_driver;
+
+static const struct apple_soc_cpufreq_info soc_t8103_info = {
+ .max_pstate = 15,
+ .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
+ .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
+};
+
+static const struct apple_soc_cpufreq_info soc_t8112_info = {
+ .max_pstate = 31,
+ .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
+ .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
+};
+
+static const struct apple_soc_cpufreq_info soc_default_info = {
+ .max_pstate = 15,
+ .cur_pstate_mask = 0, /* fallback */
+};
+
+static const struct of_device_id apple_soc_cpufreq_of_match[] = {
+ {
+ .compatible = "apple,t8103-cluster-cpufreq",
+ .data = &soc_t8103_info,
+ },
+ {
+ .compatible = "apple,t8112-cluster-cpufreq",
+ .data = &soc_t8112_info,
+ },
+ {
+ .compatible = "apple,cluster-cpufreq",
+ .data = &soc_default_info,
+ },
+ {}
+};
+
+static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+ struct apple_cpu_priv *priv = policy->driver_data;
+ struct cpufreq_frequency_table *p;
+ unsigned int pstate;
+
+ if (priv->info->cur_pstate_mask) {
+ u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
+
+ pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift;
+ } else {
+ /*
+ * For the fallback case we might not know the layout of DVFS_STATUS,
+ * so just use the command register value (which ignores boost limitations).
+ */
+ u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
+
+ pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
+ }
+
+ cpufreq_for_each_valid_entry(p, policy->freq_table)
+ if (p->driver_data == pstate)
+ return p->frequency;
+
+ dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
+ pstate);
+ return 0;
+}
+
+static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct apple_cpu_priv *priv = policy->driver_data;
+ unsigned int pstate = policy->freq_table[index].driver_data;
+ u64 reg;
+
+ /* Fallback for newer SoCs */
+ if (index > priv->info->max_pstate)
+ index = priv->info->max_pstate;
+
+ if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
+ !(reg & APPLE_DVFS_CMD_BUSY), 2,
+ APPLE_DVFS_TRANSITION_TIMEOUT)) {
+ return -EIO;
+ }
+
+ reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2);
+ reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate);
+ reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
+ reg |= APPLE_DVFS_CMD_SET;
+
+ writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
+
+ return 0;
+}
+
+static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
+ return 0;
+
+ return policy->freq_table[policy->cached_resolved_idx].frequency;
+}
+
+static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
+ void __iomem **reg_base,
+ const struct apple_soc_cpufreq_info **info)
+{
+ struct of_phandle_args args;
+ const struct of_device_id *match;
+ int ret = 0;
+
+ ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
+ "#performance-domain-cells",
+ policy->cpus, &args);
+ if (ret < 0)
+ return ret;
+
+ match = of_match_node(apple_soc_cpufreq_of_match, args.np);
+ of_node_put(args.np);
+ if (!match)
+ return -ENODEV;
+
+ *info = match->data;
+
+ *reg_base = of_iomap(args.np, 0);
+ if (!*reg_base)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {
+ &cpufreq_freq_attr_scaling_available_freqs,
+ NULL, /* Filled in below if boost is enabled */
+ NULL,
+};
+
+static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
+{
+ int ret, i;
+ unsigned int transition_latency;
+ void __iomem *reg_base;
+ struct device *cpu_dev;
+ struct apple_cpu_priv *priv;
+ const struct apple_soc_cpufreq_info *info;
+ struct cpufreq_frequency_table *freq_table;
+
+ cpu_dev = get_cpu_device(policy->cpu);
+ if (!cpu_dev) {
+ pr_err("failed to get cpu%d device\n", policy->cpu);
+ return -ENODEV;
+ }
+
+ ret = dev_pm_opp_of_add_table(cpu_dev);
+ if (ret < 0) {
+ dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
+ return ret;
+ }
+
+ ret = apple_soc_cpufreq_find_cluster(policy, &reg_base, &info);
+ if (ret) {
+ dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
+ return ret;
+ }
+
+ ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
+ if (ret) {
+ dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
+ goto out_iounmap;
+ }
+
+ ret = dev_pm_opp_get_opp_count(cpu_dev);
+ if (ret <= 0) {
+ dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
+ ret = -EPROBE_DEFER;
+ goto out_free_opp;
+ }
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv) {
+ ret = -ENOMEM;
+ goto out_free_opp;
+ }
+
+ ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
+ if (ret) {
+ dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
+ goto out_free_priv;
+ }
+
+ /* Get OPP levels (p-state indexes) and stash them in driver_data */
+ for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
+ unsigned long rate = freq_table[i].frequency * 1000 + 999;
+ struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
+
+ if (IS_ERR(opp)) {
+ ret = PTR_ERR(opp);
+ goto out_free_cpufreq_table;
+ }
+ freq_table[i].driver_data = dev_pm_opp_get_level(opp);
+ dev_pm_opp_put(opp);
+ }
+
+ priv->cpu_dev = cpu_dev;
+ priv->reg_base = reg_base;
+ priv->info = info;
+ policy->driver_data = priv;
+ policy->freq_table = freq_table;
+
+ transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
+ if (!transition_latency)
+ transition_latency = CPUFREQ_ETERNAL;
+
+ policy->cpuinfo.transition_latency = transition_latency;
+ policy->dvfs_possible_from_any_cpu = true;
+ policy->fast_switch_possible = true;
+ policy->suspend_freq = freq_table[0].frequency;
+
+ if (policy_has_boost_freq(policy)) {
+ ret = cpufreq_enable_boost_support();
+ if (ret) {
+ dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
+ } else {
+ apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
+ apple_soc_cpufreq_driver.boost_enabled = true;
+ }
+ }
+
+ return 0;
+
+out_free_cpufreq_table:
+ dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
+out_free_priv:
+ kfree(priv);
+out_free_opp:
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+out_iounmap:
+ iounmap(reg_base);
+ return ret;
+}
+
+static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ struct apple_cpu_priv *priv = policy->driver_data;
+
+ dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
+ dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
+ iounmap(priv->reg_base);
+ kfree(priv);
+
+ return 0;
+}
+
+static struct cpufreq_driver apple_soc_cpufreq_driver = {
+ .name = "apple-cpufreq",
+ .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .get = apple_soc_cpufreq_get_rate,
+ .init = apple_soc_cpufreq_init,
+ .exit = apple_soc_cpufreq_exit,
+ .target_index = apple_soc_cpufreq_set_target,
+ .fast_switch = apple_soc_cpufreq_fast_switch,
+ .register_em = cpufreq_register_em_with_opp,
+ .attr = apple_soc_cpufreq_hw_attr,
+ .suspend = cpufreq_generic_suspend,
+};
+
+static int __init apple_soc_cpufreq_module_init(void)
+{
+ if (!of_machine_is_compatible("apple,arm-platform"))
+ return -ENODEV;
+
+ return cpufreq_register_driver(&apple_soc_cpufreq_driver);
+}
+module_init(apple_soc_cpufreq_module_init);
+
+static void __exit apple_soc_cpufreq_module_exit(void)
+{
+ cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
+}
+module_exit(apple_soc_cpufreq_module_exit);
+
+MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
+MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
+MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c
new file mode 100644
index 0000000000..bea41ccabf
--- /dev/null
+++ b/drivers/cpufreq/armada-37xx-cpufreq.c
@@ -0,0 +1,562 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * CPU frequency scaling support for Armada 37xx platform.
+ *
+ * Copyright (C) 2017 Marvell
+ *
+ * Gregory CLEMENT <gregory.clement@free-electrons.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include "cpufreq-dt.h"
+
+/* Clk register set */
+#define ARMADA_37XX_CLK_TBG_SEL 0
+#define ARMADA_37XX_CLK_TBG_SEL_CPU_OFF 22
+
+/* Power management in North Bridge register set */
+#define ARMADA_37XX_NB_L0L1 0x18
+#define ARMADA_37XX_NB_L2L3 0x1C
+#define ARMADA_37XX_NB_TBG_DIV_OFF 13
+#define ARMADA_37XX_NB_TBG_DIV_MASK 0x7
+#define ARMADA_37XX_NB_CLK_SEL_OFF 11
+#define ARMADA_37XX_NB_CLK_SEL_MASK 0x1
+#define ARMADA_37XX_NB_CLK_SEL_TBG 0x1
+#define ARMADA_37XX_NB_TBG_SEL_OFF 9
+#define ARMADA_37XX_NB_TBG_SEL_MASK 0x3
+#define ARMADA_37XX_NB_VDD_SEL_OFF 6
+#define ARMADA_37XX_NB_VDD_SEL_MASK 0x3
+#define ARMADA_37XX_NB_CONFIG_SHIFT 16
+#define ARMADA_37XX_NB_DYN_MOD 0x24
+#define ARMADA_37XX_NB_CLK_SEL_EN BIT(26)
+#define ARMADA_37XX_NB_TBG_EN BIT(28)
+#define ARMADA_37XX_NB_DIV_EN BIT(29)
+#define ARMADA_37XX_NB_VDD_EN BIT(30)
+#define ARMADA_37XX_NB_DFS_EN BIT(31)
+#define ARMADA_37XX_NB_CPU_LOAD 0x30
+#define ARMADA_37XX_NB_CPU_LOAD_MASK 0x3
+#define ARMADA_37XX_DVFS_LOAD_0 0
+#define ARMADA_37XX_DVFS_LOAD_1 1
+#define ARMADA_37XX_DVFS_LOAD_2 2
+#define ARMADA_37XX_DVFS_LOAD_3 3
+
+/* AVS register set */
+#define ARMADA_37XX_AVS_CTL0 0x0
+#define ARMADA_37XX_AVS_ENABLE BIT(30)
+#define ARMADA_37XX_AVS_HIGH_VDD_LIMIT 16
+#define ARMADA_37XX_AVS_LOW_VDD_LIMIT 22
+#define ARMADA_37XX_AVS_VDD_MASK 0x3F
+#define ARMADA_37XX_AVS_CTL2 0x8
+#define ARMADA_37XX_AVS_LOW_VDD_EN BIT(6)
+#define ARMADA_37XX_AVS_VSET(x) (0x1C + 4 * (x))
+
+/*
+ * On Armada 37xx the Power management manages 4 level of CPU load,
+ * each level can be associated with a CPU clock source, a CPU
+ * divider, a VDD level, etc...
+ */
+#define LOAD_LEVEL_NR 4
+
+#define MIN_VOLT_MV 1000
+#define MIN_VOLT_MV_FOR_L1_1000MHZ 1108
+#define MIN_VOLT_MV_FOR_L1_1200MHZ 1155
+
+/* AVS value for the corresponding voltage (in mV) */
+static int avs_map[] = {
+ 747, 758, 770, 782, 793, 805, 817, 828, 840, 852, 863, 875, 887, 898,
+ 910, 922, 933, 945, 957, 968, 980, 992, 1003, 1015, 1027, 1038, 1050,
+ 1062, 1073, 1085, 1097, 1108, 1120, 1132, 1143, 1155, 1167, 1178, 1190,
+ 1202, 1213, 1225, 1237, 1248, 1260, 1272, 1283, 1295, 1307, 1318, 1330,
+ 1342
+};
+
+struct armada37xx_cpufreq_state {
+ struct platform_device *pdev;
+ struct device *cpu_dev;
+ struct regmap *regmap;
+ u32 nb_l0l1;
+ u32 nb_l2l3;
+ u32 nb_dyn_mod;
+ u32 nb_cpu_load;
+};
+
+static struct armada37xx_cpufreq_state *armada37xx_cpufreq_state;
+
+struct armada_37xx_dvfs {
+ u32 cpu_freq_max;
+ u8 divider[LOAD_LEVEL_NR];
+ u32 avs[LOAD_LEVEL_NR];
+};
+
+static struct armada_37xx_dvfs armada_37xx_dvfs[] = {
+ /*
+ * The cpufreq scaling for 1.2 GHz variant of the SOC is currently
+ * unstable because we do not know how to configure it properly.
+ */
+ /* {.cpu_freq_max = 1200*1000*1000, .divider = {1, 2, 4, 6} }, */
+ {.cpu_freq_max = 1000*1000*1000, .divider = {1, 2, 4, 5} },
+ {.cpu_freq_max = 800*1000*1000, .divider = {1, 2, 3, 4} },
+ {.cpu_freq_max = 600*1000*1000, .divider = {2, 4, 5, 6} },
+};
+
+static struct armada_37xx_dvfs *armada_37xx_cpu_freq_info_get(u32 freq)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(armada_37xx_dvfs); i++) {
+ if (freq == armada_37xx_dvfs[i].cpu_freq_max)
+ return &armada_37xx_dvfs[i];
+ }
+
+ pr_err("Unsupported CPU frequency %d MHz\n", freq/1000000);
+ return NULL;
+}
+
+/*
+ * Setup the four level managed by the hardware. Once the four level
+ * will be configured then the DVFS will be enabled.
+ */
+static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base,
+ struct regmap *clk_base, u8 *divider)
+{
+ u32 cpu_tbg_sel;
+ int load_lvl;
+
+ /* Determine to which TBG clock is CPU connected */
+ regmap_read(clk_base, ARMADA_37XX_CLK_TBG_SEL, &cpu_tbg_sel);
+ cpu_tbg_sel >>= ARMADA_37XX_CLK_TBG_SEL_CPU_OFF;
+ cpu_tbg_sel &= ARMADA_37XX_NB_TBG_SEL_MASK;
+
+ for (load_lvl = 0; load_lvl < LOAD_LEVEL_NR; load_lvl++) {
+ unsigned int reg, mask, val, offset = 0;
+
+ if (load_lvl <= ARMADA_37XX_DVFS_LOAD_1)
+ reg = ARMADA_37XX_NB_L0L1;
+ else
+ reg = ARMADA_37XX_NB_L2L3;
+
+ if (load_lvl == ARMADA_37XX_DVFS_LOAD_0 ||
+ load_lvl == ARMADA_37XX_DVFS_LOAD_2)
+ offset += ARMADA_37XX_NB_CONFIG_SHIFT;
+
+ /* Set cpu clock source, for all the level we use TBG */
+ val = ARMADA_37XX_NB_CLK_SEL_TBG << ARMADA_37XX_NB_CLK_SEL_OFF;
+ mask = (ARMADA_37XX_NB_CLK_SEL_MASK
+ << ARMADA_37XX_NB_CLK_SEL_OFF);
+
+ /* Set TBG index, for all levels we use the same TBG */
+ val = cpu_tbg_sel << ARMADA_37XX_NB_TBG_SEL_OFF;
+ mask = (ARMADA_37XX_NB_TBG_SEL_MASK
+ << ARMADA_37XX_NB_TBG_SEL_OFF);
+
+ /*
+ * Set cpu divider based on the pre-computed array in
+ * order to have balanced step.
+ */
+ val |= divider[load_lvl] << ARMADA_37XX_NB_TBG_DIV_OFF;
+ mask |= (ARMADA_37XX_NB_TBG_DIV_MASK
+ << ARMADA_37XX_NB_TBG_DIV_OFF);
+
+ /* Set VDD divider which is actually the load level. */
+ val |= load_lvl << ARMADA_37XX_NB_VDD_SEL_OFF;
+ mask |= (ARMADA_37XX_NB_VDD_SEL_MASK
+ << ARMADA_37XX_NB_VDD_SEL_OFF);
+
+ val <<= offset;
+ mask <<= offset;
+
+ regmap_update_bits(base, reg, mask, val);
+ }
+}
+
+/*
+ * Find out the armada 37x supported AVS value whose voltage value is
+ * the round-up closest to the target voltage value.
+ */
+static u32 armada_37xx_avs_val_match(int target_vm)
+{
+ u32 avs;
+
+ /* Find out the round-up closest supported voltage value */
+ for (avs = 0; avs < ARRAY_SIZE(avs_map); avs++)
+ if (avs_map[avs] >= target_vm)
+ break;
+
+ /*
+ * If all supported voltages are smaller than target one,
+ * choose the largest supported voltage
+ */
+ if (avs == ARRAY_SIZE(avs_map))
+ avs = ARRAY_SIZE(avs_map) - 1;
+
+ return avs;
+}
+
+/*
+ * For Armada 37xx soc, L0(VSET0) VDD AVS value is set to SVC revision
+ * value or a default value when SVC is not supported.
+ * - L0 can be read out from the register of AVS_CTRL_0 and L0 voltage
+ * can be got from the mapping table of avs_map.
+ * - L1 voltage should be about 100mv smaller than L0 voltage
+ * - L2 & L3 voltage should be about 150mv smaller than L0 voltage.
+ * This function calculates L1 & L2 & L3 AVS values dynamically based
+ * on L0 voltage and fill all AVS values to the AVS value table.
+ * When base CPU frequency is 1000 or 1200 MHz then there is additional
+ * minimal avs value for load L1.
+ */
+static void __init armada37xx_cpufreq_avs_configure(struct regmap *base,
+ struct armada_37xx_dvfs *dvfs)
+{
+ unsigned int target_vm;
+ int load_level = 0;
+ u32 l0_vdd_min;
+
+ if (base == NULL)
+ return;
+
+ /* Get L0 VDD min value */
+ regmap_read(base, ARMADA_37XX_AVS_CTL0, &l0_vdd_min);
+ l0_vdd_min = (l0_vdd_min >> ARMADA_37XX_AVS_LOW_VDD_LIMIT) &
+ ARMADA_37XX_AVS_VDD_MASK;
+ if (l0_vdd_min >= ARRAY_SIZE(avs_map)) {
+ pr_err("L0 VDD MIN %d is not correct.\n", l0_vdd_min);
+ return;
+ }
+ dvfs->avs[0] = l0_vdd_min;
+
+ if (avs_map[l0_vdd_min] <= MIN_VOLT_MV) {
+ /*
+ * If L0 voltage is smaller than 1000mv, then all VDD sets
+ * use L0 voltage;
+ */
+ u32 avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV);
+
+ for (load_level = 1; load_level < LOAD_LEVEL_NR; load_level++)
+ dvfs->avs[load_level] = avs_min;
+
+ /*
+ * Set the avs values for load L0 and L1 when base CPU frequency
+ * is 1000/1200 MHz to its typical initial values according to
+ * the Armada 3700 Hardware Specifications.
+ */
+ if (dvfs->cpu_freq_max >= 1000*1000*1000) {
+ if (dvfs->cpu_freq_max >= 1200*1000*1000)
+ avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1200MHZ);
+ else
+ avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1000MHZ);
+ dvfs->avs[0] = dvfs->avs[1] = avs_min;
+ }
+
+ return;
+ }
+
+ /*
+ * L1 voltage is equal to L0 voltage - 100mv and it must be
+ * larger than 1000mv
+ */
+
+ target_vm = avs_map[l0_vdd_min] - 100;
+ target_vm = target_vm > MIN_VOLT_MV ? target_vm : MIN_VOLT_MV;
+ dvfs->avs[1] = armada_37xx_avs_val_match(target_vm);
+
+ /*
+ * L2 & L3 voltage is equal to L0 voltage - 150mv and it must
+ * be larger than 1000mv
+ */
+ target_vm = avs_map[l0_vdd_min] - 150;
+ target_vm = target_vm > MIN_VOLT_MV ? target_vm : MIN_VOLT_MV;
+ dvfs->avs[2] = dvfs->avs[3] = armada_37xx_avs_val_match(target_vm);
+
+ /*
+ * Fix the avs value for load L1 when base CPU frequency is 1000/1200 MHz,
+ * otherwise the CPU gets stuck when switching from load L1 to load L0.
+ * Also ensure that avs value for load L1 is not higher than for L0.
+ */
+ if (dvfs->cpu_freq_max >= 1000*1000*1000) {
+ u32 avs_min_l1;
+
+ if (dvfs->cpu_freq_max >= 1200*1000*1000)
+ avs_min_l1 = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1200MHZ);
+ else
+ avs_min_l1 = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1000MHZ);
+
+ if (avs_min_l1 > dvfs->avs[0])
+ avs_min_l1 = dvfs->avs[0];
+
+ if (dvfs->avs[1] < avs_min_l1)
+ dvfs->avs[1] = avs_min_l1;
+ }
+}
+
+static void __init armada37xx_cpufreq_avs_setup(struct regmap *base,
+ struct armada_37xx_dvfs *dvfs)
+{
+ unsigned int avs_val = 0;
+ int load_level = 0;
+
+ if (base == NULL)
+ return;
+
+ /* Disable AVS before the configuration */
+ regmap_update_bits(base, ARMADA_37XX_AVS_CTL0,
+ ARMADA_37XX_AVS_ENABLE, 0);
+
+
+ /* Enable low voltage mode */
+ regmap_update_bits(base, ARMADA_37XX_AVS_CTL2,
+ ARMADA_37XX_AVS_LOW_VDD_EN,
+ ARMADA_37XX_AVS_LOW_VDD_EN);
+
+
+ for (load_level = 1; load_level < LOAD_LEVEL_NR; load_level++) {
+ avs_val = dvfs->avs[load_level];
+ regmap_update_bits(base, ARMADA_37XX_AVS_VSET(load_level-1),
+ ARMADA_37XX_AVS_VDD_MASK << ARMADA_37XX_AVS_HIGH_VDD_LIMIT |
+ ARMADA_37XX_AVS_VDD_MASK << ARMADA_37XX_AVS_LOW_VDD_LIMIT,
+ avs_val << ARMADA_37XX_AVS_HIGH_VDD_LIMIT |
+ avs_val << ARMADA_37XX_AVS_LOW_VDD_LIMIT);
+ }
+
+ /* Enable AVS after the configuration */
+ regmap_update_bits(base, ARMADA_37XX_AVS_CTL0,
+ ARMADA_37XX_AVS_ENABLE,
+ ARMADA_37XX_AVS_ENABLE);
+
+}
+
+static void armada37xx_cpufreq_disable_dvfs(struct regmap *base)
+{
+ unsigned int reg = ARMADA_37XX_NB_DYN_MOD,
+ mask = ARMADA_37XX_NB_DFS_EN;
+
+ regmap_update_bits(base, reg, mask, 0);
+}
+
+static void __init armada37xx_cpufreq_enable_dvfs(struct regmap *base)
+{
+ unsigned int val, reg = ARMADA_37XX_NB_CPU_LOAD,
+ mask = ARMADA_37XX_NB_CPU_LOAD_MASK;
+
+ /* Start with the highest load (0) */
+ val = ARMADA_37XX_DVFS_LOAD_0;
+ regmap_update_bits(base, reg, mask, val);
+
+ /* Now enable DVFS for the CPUs */
+ reg = ARMADA_37XX_NB_DYN_MOD;
+ mask = ARMADA_37XX_NB_CLK_SEL_EN | ARMADA_37XX_NB_TBG_EN |
+ ARMADA_37XX_NB_DIV_EN | ARMADA_37XX_NB_VDD_EN |
+ ARMADA_37XX_NB_DFS_EN;
+
+ regmap_update_bits(base, reg, mask, mask);
+}
+
+static int armada37xx_cpufreq_suspend(struct cpufreq_policy *policy)
+{
+ struct armada37xx_cpufreq_state *state = armada37xx_cpufreq_state;
+
+ regmap_read(state->regmap, ARMADA_37XX_NB_L0L1, &state->nb_l0l1);
+ regmap_read(state->regmap, ARMADA_37XX_NB_L2L3, &state->nb_l2l3);
+ regmap_read(state->regmap, ARMADA_37XX_NB_CPU_LOAD,
+ &state->nb_cpu_load);
+ regmap_read(state->regmap, ARMADA_37XX_NB_DYN_MOD, &state->nb_dyn_mod);
+
+ return 0;
+}
+
+static int armada37xx_cpufreq_resume(struct cpufreq_policy *policy)
+{
+ struct armada37xx_cpufreq_state *state = armada37xx_cpufreq_state;
+
+ /* Ensure DVFS is disabled otherwise the following registers are RO */
+ armada37xx_cpufreq_disable_dvfs(state->regmap);
+
+ regmap_write(state->regmap, ARMADA_37XX_NB_L0L1, state->nb_l0l1);
+ regmap_write(state->regmap, ARMADA_37XX_NB_L2L3, state->nb_l2l3);
+ regmap_write(state->regmap, ARMADA_37XX_NB_CPU_LOAD,
+ state->nb_cpu_load);
+
+ /*
+ * NB_DYN_MOD register is the one that actually enable back DVFS if it
+ * was enabled before the suspend operation. This must be done last
+ * otherwise other registers are not writable.
+ */
+ regmap_write(state->regmap, ARMADA_37XX_NB_DYN_MOD, state->nb_dyn_mod);
+
+ return 0;
+}
+
+static int __init armada37xx_cpufreq_driver_init(void)
+{
+ struct cpufreq_dt_platform_data pdata;
+ struct armada_37xx_dvfs *dvfs;
+ struct platform_device *pdev;
+ unsigned long freq;
+ unsigned int base_frequency;
+ struct regmap *nb_clk_base, *nb_pm_base, *avs_base;
+ struct device *cpu_dev;
+ int load_lvl, ret;
+ struct clk *clk, *parent;
+
+ nb_clk_base =
+ syscon_regmap_lookup_by_compatible("marvell,armada-3700-periph-clock-nb");
+ if (IS_ERR(nb_clk_base))
+ return -ENODEV;
+
+ nb_pm_base =
+ syscon_regmap_lookup_by_compatible("marvell,armada-3700-nb-pm");
+
+ if (IS_ERR(nb_pm_base))
+ return -ENODEV;
+
+ avs_base =
+ syscon_regmap_lookup_by_compatible("marvell,armada-3700-avs");
+
+ /* if AVS is not present don't use it but still try to setup dvfs */
+ if (IS_ERR(avs_base)) {
+ pr_info("Syscon failed for Adapting Voltage Scaling: skip it\n");
+ avs_base = NULL;
+ }
+ /* Before doing any configuration on the DVFS first, disable it */
+ armada37xx_cpufreq_disable_dvfs(nb_pm_base);
+
+ /*
+ * On CPU 0 register the operating points supported (which are
+ * the nominal CPU frequency and full integer divisions of
+ * it).
+ */
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev) {
+ dev_err(cpu_dev, "Cannot get CPU\n");
+ return -ENODEV;
+ }
+
+ clk = clk_get(cpu_dev, NULL);
+ if (IS_ERR(clk)) {
+ dev_err(cpu_dev, "Cannot get clock for CPU0\n");
+ return PTR_ERR(clk);
+ }
+
+ parent = clk_get_parent(clk);
+ if (IS_ERR(parent)) {
+ dev_err(cpu_dev, "Cannot get parent clock for CPU0\n");
+ clk_put(clk);
+ return PTR_ERR(parent);
+ }
+
+ /* Get parent CPU frequency */
+ base_frequency = clk_get_rate(parent);
+
+ if (!base_frequency) {
+ dev_err(cpu_dev, "Failed to get parent clock rate for CPU\n");
+ clk_put(clk);
+ return -EINVAL;
+ }
+
+ dvfs = armada_37xx_cpu_freq_info_get(base_frequency);
+ if (!dvfs) {
+ clk_put(clk);
+ return -EINVAL;
+ }
+
+ armada37xx_cpufreq_state = kmalloc(sizeof(*armada37xx_cpufreq_state),
+ GFP_KERNEL);
+ if (!armada37xx_cpufreq_state) {
+ clk_put(clk);
+ return -ENOMEM;
+ }
+
+ armada37xx_cpufreq_state->regmap = nb_pm_base;
+
+ armada37xx_cpufreq_avs_configure(avs_base, dvfs);
+ armada37xx_cpufreq_avs_setup(avs_base, dvfs);
+
+ armada37xx_cpufreq_dvfs_setup(nb_pm_base, nb_clk_base, dvfs->divider);
+ clk_put(clk);
+
+ for (load_lvl = ARMADA_37XX_DVFS_LOAD_0; load_lvl < LOAD_LEVEL_NR;
+ load_lvl++) {
+ unsigned long u_volt = avs_map[dvfs->avs[load_lvl]] * 1000;
+ freq = base_frequency / dvfs->divider[load_lvl];
+ ret = dev_pm_opp_add(cpu_dev, freq, u_volt);
+ if (ret)
+ goto remove_opp;
+
+
+ }
+
+ /* Now that everything is setup, enable the DVFS at hardware level */
+ armada37xx_cpufreq_enable_dvfs(nb_pm_base);
+
+ memset(&pdata, 0, sizeof(pdata));
+ pdata.suspend = armada37xx_cpufreq_suspend;
+ pdata.resume = armada37xx_cpufreq_resume;
+
+ pdev = platform_device_register_data(NULL, "cpufreq-dt", -1, &pdata,
+ sizeof(pdata));
+ ret = PTR_ERR_OR_ZERO(pdev);
+ if (ret)
+ goto disable_dvfs;
+
+ armada37xx_cpufreq_state->cpu_dev = cpu_dev;
+ armada37xx_cpufreq_state->pdev = pdev;
+ platform_set_drvdata(pdev, dvfs);
+ return 0;
+
+disable_dvfs:
+ armada37xx_cpufreq_disable_dvfs(nb_pm_base);
+remove_opp:
+ /* clean-up the already added opp before leaving */
+ while (load_lvl-- > ARMADA_37XX_DVFS_LOAD_0) {
+ freq = base_frequency / dvfs->divider[load_lvl];
+ dev_pm_opp_remove(cpu_dev, freq);
+ }
+
+ kfree(armada37xx_cpufreq_state);
+
+ return ret;
+}
+/* late_initcall, to guarantee the driver is loaded after A37xx clock driver */
+late_initcall(armada37xx_cpufreq_driver_init);
+
+static void __exit armada37xx_cpufreq_driver_exit(void)
+{
+ struct platform_device *pdev = armada37xx_cpufreq_state->pdev;
+ struct armada_37xx_dvfs *dvfs = platform_get_drvdata(pdev);
+ unsigned long freq;
+ int load_lvl;
+
+ platform_device_unregister(pdev);
+
+ armada37xx_cpufreq_disable_dvfs(armada37xx_cpufreq_state->regmap);
+
+ for (load_lvl = ARMADA_37XX_DVFS_LOAD_0; load_lvl < LOAD_LEVEL_NR; load_lvl++) {
+ freq = dvfs->cpu_freq_max / dvfs->divider[load_lvl];
+ dev_pm_opp_remove(armada37xx_cpufreq_state->cpu_dev, freq);
+ }
+
+ kfree(armada37xx_cpufreq_state);
+}
+module_exit(armada37xx_cpufreq_driver_exit);
+
+static const struct of_device_id __maybe_unused armada37xx_cpufreq_of_match[] = {
+ { .compatible = "marvell,armada-3700-nb-pm" },
+ { },
+};
+MODULE_DEVICE_TABLE(of, armada37xx_cpufreq_of_match);
+
+MODULE_AUTHOR("Gregory CLEMENT <gregory.clement@free-electrons.com>");
+MODULE_DESCRIPTION("Armada 37xx cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/armada-8k-cpufreq.c b/drivers/cpufreq/armada-8k-cpufreq.c
new file mode 100644
index 0000000000..8afefdea4d
--- /dev/null
+++ b/drivers/cpufreq/armada-8k-cpufreq.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * CPUFreq support for Armada 8K
+ *
+ * Copyright (C) 2018 Marvell
+ *
+ * Omri Itach <omrii@marvell.com>
+ * Gregory Clement <gregory.clement@bootlin.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+
+static const struct of_device_id __maybe_unused armada_8k_cpufreq_of_match[] = {
+ { .compatible = "marvell,ap806-cpu-clock" },
+ { .compatible = "marvell,ap807-cpu-clock" },
+ { },
+};
+MODULE_DEVICE_TABLE(of, armada_8k_cpufreq_of_match);
+
+/*
+ * Setup the opps list with the divider for the max frequency, that
+ * will be filled at runtime.
+ */
+static const int opps_div[] __initconst = {1, 2, 3, 4};
+
+static struct platform_device *armada_8k_pdev;
+
+struct freq_table {
+ struct device *cpu_dev;
+ unsigned int freq[ARRAY_SIZE(opps_div)];
+};
+
+/* If the CPUs share the same clock, then they are in the same cluster. */
+static void __init armada_8k_get_sharing_cpus(struct clk *cur_clk,
+ struct cpumask *cpumask)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct device *cpu_dev;
+ struct clk *clk;
+
+ cpu_dev = get_cpu_device(cpu);
+ if (!cpu_dev) {
+ pr_warn("Failed to get cpu%d device\n", cpu);
+ continue;
+ }
+
+ clk = clk_get(cpu_dev, 0);
+ if (IS_ERR(clk)) {
+ pr_warn("Cannot get clock for CPU %d\n", cpu);
+ } else {
+ if (clk_is_match(clk, cur_clk))
+ cpumask_set_cpu(cpu, cpumask);
+
+ clk_put(clk);
+ }
+ }
+}
+
+static int __init armada_8k_add_opp(struct clk *clk, struct device *cpu_dev,
+ struct freq_table *freq_tables,
+ int opps_index)
+{
+ unsigned int cur_frequency;
+ unsigned int freq;
+ int i, ret;
+
+ /* Get nominal (current) CPU frequency. */
+ cur_frequency = clk_get_rate(clk);
+ if (!cur_frequency) {
+ dev_err(cpu_dev, "Failed to get clock rate for this CPU\n");
+ return -EINVAL;
+ }
+
+ freq_tables[opps_index].cpu_dev = cpu_dev;
+
+ for (i = 0; i < ARRAY_SIZE(opps_div); i++) {
+ freq = cur_frequency / opps_div[i];
+
+ ret = dev_pm_opp_add(cpu_dev, freq, 0);
+ if (ret)
+ return ret;
+
+ freq_tables[opps_index].freq[i] = freq;
+ }
+
+ return 0;
+}
+
+static void armada_8k_cpufreq_free_table(struct freq_table *freq_tables)
+{
+ int opps_index, nb_cpus = num_possible_cpus();
+
+ for (opps_index = 0 ; opps_index <= nb_cpus; opps_index++) {
+ int i;
+
+ /* If cpu_dev is NULL then we reached the end of the array */
+ if (!freq_tables[opps_index].cpu_dev)
+ break;
+
+ for (i = 0; i < ARRAY_SIZE(opps_div); i++) {
+ /*
+ * A 0Hz frequency is not valid, this meant
+ * that it was not yet initialized so there is
+ * no more opp to free
+ */
+ if (freq_tables[opps_index].freq[i] == 0)
+ break;
+
+ dev_pm_opp_remove(freq_tables[opps_index].cpu_dev,
+ freq_tables[opps_index].freq[i]);
+ }
+ }
+
+ kfree(freq_tables);
+}
+
+static int __init armada_8k_cpufreq_init(void)
+{
+ int ret = 0, opps_index = 0, cpu, nb_cpus;
+ struct freq_table *freq_tables;
+ struct device_node *node;
+ struct cpumask cpus;
+
+ node = of_find_matching_node_and_match(NULL, armada_8k_cpufreq_of_match,
+ NULL);
+ if (!node || !of_device_is_available(node)) {
+ of_node_put(node);
+ return -ENODEV;
+ }
+ of_node_put(node);
+
+ nb_cpus = num_possible_cpus();
+ freq_tables = kcalloc(nb_cpus, sizeof(*freq_tables), GFP_KERNEL);
+ if (!freq_tables)
+ return -ENOMEM;
+ cpumask_copy(&cpus, cpu_possible_mask);
+
+ /*
+ * For each CPU, this loop registers the operating points
+ * supported (which are the nominal CPU frequency and full integer
+ * divisions of it).
+ */
+ for_each_cpu(cpu, &cpus) {
+ struct cpumask shared_cpus;
+ struct device *cpu_dev;
+ struct clk *clk;
+
+ cpu_dev = get_cpu_device(cpu);
+
+ if (!cpu_dev) {
+ pr_err("Cannot get CPU %d\n", cpu);
+ continue;
+ }
+
+ clk = clk_get(cpu_dev, 0);
+
+ if (IS_ERR(clk)) {
+ pr_err("Cannot get clock for CPU %d\n", cpu);
+ ret = PTR_ERR(clk);
+ goto remove_opp;
+ }
+
+ ret = armada_8k_add_opp(clk, cpu_dev, freq_tables, opps_index);
+ if (ret) {
+ clk_put(clk);
+ goto remove_opp;
+ }
+
+ opps_index++;
+ cpumask_clear(&shared_cpus);
+ armada_8k_get_sharing_cpus(clk, &shared_cpus);
+ dev_pm_opp_set_sharing_cpus(cpu_dev, &shared_cpus);
+ cpumask_andnot(&cpus, &cpus, &shared_cpus);
+ clk_put(clk);
+ }
+
+ armada_8k_pdev = platform_device_register_simple("cpufreq-dt", -1,
+ NULL, 0);
+ ret = PTR_ERR_OR_ZERO(armada_8k_pdev);
+ if (ret)
+ goto remove_opp;
+
+ platform_set_drvdata(armada_8k_pdev, freq_tables);
+
+ return 0;
+
+remove_opp:
+ armada_8k_cpufreq_free_table(freq_tables);
+ return ret;
+}
+module_init(armada_8k_cpufreq_init);
+
+static void __exit armada_8k_cpufreq_exit(void)
+{
+ struct freq_table *freq_tables = platform_get_drvdata(armada_8k_pdev);
+
+ platform_device_unregister(armada_8k_pdev);
+ armada_8k_cpufreq_free_table(freq_tables);
+}
+module_exit(armada_8k_cpufreq_exit);
+
+MODULE_AUTHOR("Gregory Clement <gregory.clement@bootlin.com>");
+MODULE_DESCRIPTION("Armada 8K cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c
new file mode 100644
index 0000000000..39221a9a18
--- /dev/null
+++ b/drivers/cpufreq/bmips-cpufreq.c
@@ -0,0 +1,189 @@
+/*
+ * CPU frequency scaling for Broadcom BMIPS SoCs
+ *
+ * Copyright (c) 2017 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+
+/* for mips_hpt_frequency */
+#include <asm/time.h>
+
+#define BMIPS_CPUFREQ_PREFIX "bmips"
+#define BMIPS_CPUFREQ_NAME BMIPS_CPUFREQ_PREFIX "-cpufreq"
+
+#define TRANSITION_LATENCY (25 * 1000) /* 25 us */
+
+#define BMIPS5_CLK_DIV_SET_SHIFT 0x7
+#define BMIPS5_CLK_DIV_SHIFT 0x4
+#define BMIPS5_CLK_DIV_MASK 0xf
+
+enum bmips_type {
+ BMIPS5000,
+ BMIPS5200,
+};
+
+struct cpufreq_compat {
+ const char *compatible;
+ unsigned int bmips_type;
+ unsigned int clk_mult;
+ unsigned int max_freqs;
+};
+
+#define BMIPS(c, t, m, f) { \
+ .compatible = c, \
+ .bmips_type = (t), \
+ .clk_mult = (m), \
+ .max_freqs = (f), \
+}
+
+static struct cpufreq_compat bmips_cpufreq_compat[] = {
+ BMIPS("brcm,bmips5000", BMIPS5000, 8, 4),
+ BMIPS("brcm,bmips5200", BMIPS5200, 8, 4),
+ { }
+};
+
+static struct cpufreq_compat *priv;
+
+static int htp_freq_to_cpu_freq(unsigned int clk_mult)
+{
+ return mips_hpt_frequency * clk_mult / 1000;
+}
+
+static struct cpufreq_frequency_table *
+bmips_cpufreq_get_freq_table(const struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *table;
+ unsigned long cpu_freq;
+ int i;
+
+ cpu_freq = htp_freq_to_cpu_freq(priv->clk_mult);
+
+ table = kmalloc_array(priv->max_freqs + 1, sizeof(*table), GFP_KERNEL);
+ if (!table)
+ return ERR_PTR(-ENOMEM);
+
+ for (i = 0; i < priv->max_freqs; i++) {
+ table[i].frequency = cpu_freq / (1 << i);
+ table[i].driver_data = i;
+ }
+ table[i].frequency = CPUFREQ_TABLE_END;
+
+ return table;
+}
+
+static unsigned int bmips_cpufreq_get(unsigned int cpu)
+{
+ unsigned int div;
+ uint32_t mode;
+
+ switch (priv->bmips_type) {
+ case BMIPS5200:
+ case BMIPS5000:
+ mode = read_c0_brcm_mode();
+ div = ((mode >> BMIPS5_CLK_DIV_SHIFT) & BMIPS5_CLK_DIV_MASK);
+ break;
+ default:
+ div = 0;
+ }
+
+ return htp_freq_to_cpu_freq(priv->clk_mult) / (1 << div);
+}
+
+static int bmips_cpufreq_target_index(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ unsigned int div = policy->freq_table[index].driver_data;
+
+ switch (priv->bmips_type) {
+ case BMIPS5200:
+ case BMIPS5000:
+ change_c0_brcm_mode(BMIPS5_CLK_DIV_MASK << BMIPS5_CLK_DIV_SHIFT,
+ (1 << BMIPS5_CLK_DIV_SET_SHIFT) |
+ (div << BMIPS5_CLK_DIV_SHIFT));
+ break;
+ default:
+ return -ENOTSUPP;
+ }
+
+ return 0;
+}
+
+static int bmips_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ kfree(policy->freq_table);
+
+ return 0;
+}
+
+static int bmips_cpufreq_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *freq_table;
+
+ freq_table = bmips_cpufreq_get_freq_table(policy);
+ if (IS_ERR(freq_table)) {
+ pr_err("%s: couldn't determine frequency table (%ld).\n",
+ BMIPS_CPUFREQ_NAME, PTR_ERR(freq_table));
+ return PTR_ERR(freq_table);
+ }
+
+ cpufreq_generic_init(policy, freq_table, TRANSITION_LATENCY);
+ pr_info("%s: registered\n", BMIPS_CPUFREQ_NAME);
+
+ return 0;
+}
+
+static struct cpufreq_driver bmips_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = bmips_cpufreq_target_index,
+ .get = bmips_cpufreq_get,
+ .init = bmips_cpufreq_init,
+ .exit = bmips_cpufreq_exit,
+ .attr = cpufreq_generic_attr,
+ .name = BMIPS_CPUFREQ_PREFIX,
+};
+
+static int __init bmips_cpufreq_driver_init(void)
+{
+ struct cpufreq_compat *cc;
+ struct device_node *np;
+
+ for (cc = bmips_cpufreq_compat; cc->compatible; cc++) {
+ np = of_find_compatible_node(NULL, "cpu", cc->compatible);
+ if (np) {
+ of_node_put(np);
+ priv = cc;
+ break;
+ }
+ }
+
+ /* We hit the guard element of the array. No compatible CPU found. */
+ if (!cc->compatible)
+ return -ENODEV;
+
+ return cpufreq_register_driver(&bmips_cpufreq_driver);
+}
+module_init(bmips_cpufreq_driver_init);
+
+static void __exit bmips_cpufreq_driver_exit(void)
+{
+ cpufreq_unregister_driver(&bmips_cpufreq_driver);
+}
+module_exit(bmips_cpufreq_driver_exit);
+
+MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>");
+MODULE_DESCRIPTION("CPUfreq driver for Broadcom BMIPS SoCs");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c
new file mode 100644
index 0000000000..35fb3a559e
--- /dev/null
+++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c
@@ -0,0 +1,781 @@
+/*
+ * CPU frequency scaling for Broadcom SoCs with AVS firmware that
+ * supports DVS or DVFS
+ *
+ * Copyright (c) 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+/*
+ * "AVS" is the name of a firmware developed at Broadcom. It derives
+ * its name from the technique called "Adaptive Voltage Scaling".
+ * Adaptive voltage scaling was the original purpose of this firmware.
+ * The AVS firmware still supports "AVS mode", where all it does is
+ * adaptive voltage scaling. However, on some newer Broadcom SoCs, the
+ * AVS Firmware, despite its unchanged name, also supports DFS mode and
+ * DVFS mode.
+ *
+ * In the context of this document and the related driver, "AVS" by
+ * itself always means the Broadcom firmware and never refers to the
+ * technique called "Adaptive Voltage Scaling".
+ *
+ * The Broadcom STB AVS CPUfreq driver provides voltage and frequency
+ * scaling on Broadcom SoCs using AVS firmware with support for DFS and
+ * DVFS. The AVS firmware is running on its own co-processor. The
+ * driver supports both uniprocessor (UP) and symmetric multiprocessor
+ * (SMP) systems which share clock and voltage across all CPUs.
+ *
+ * Actual voltage and frequency scaling is done solely by the AVS
+ * firmware. This driver does not change frequency or voltage itself.
+ * It provides a standard CPUfreq interface to the rest of the kernel
+ * and to userland. It interfaces with the AVS firmware to effect the
+ * requested changes and to report back the current system status in a
+ * way that is expected by existing tools.
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/semaphore.h>
+
+/* Max number of arguments AVS calls take */
+#define AVS_MAX_CMD_ARGS 4
+/*
+ * This macro is used to generate AVS parameter register offsets. For
+ * x >= AVS_MAX_CMD_ARGS, it returns 0 to protect against accidental memory
+ * access outside of the parameter range. (Offset 0 is the first parameter.)
+ */
+#define AVS_PARAM_MULT(x) ((x) < AVS_MAX_CMD_ARGS ? (x) : 0)
+
+/* AVS Mailbox Register offsets */
+#define AVS_MBOX_COMMAND 0x00
+#define AVS_MBOX_STATUS 0x04
+#define AVS_MBOX_VOLTAGE0 0x08
+#define AVS_MBOX_TEMP0 0x0c
+#define AVS_MBOX_PV0 0x10
+#define AVS_MBOX_MV0 0x14
+#define AVS_MBOX_PARAM(x) (0x18 + AVS_PARAM_MULT(x) * sizeof(u32))
+#define AVS_MBOX_REVISION 0x28
+#define AVS_MBOX_PSTATE 0x2c
+#define AVS_MBOX_HEARTBEAT 0x30
+#define AVS_MBOX_MAGIC 0x34
+#define AVS_MBOX_SIGMA_HVT 0x38
+#define AVS_MBOX_SIGMA_SVT 0x3c
+#define AVS_MBOX_VOLTAGE1 0x40
+#define AVS_MBOX_TEMP1 0x44
+#define AVS_MBOX_PV1 0x48
+#define AVS_MBOX_MV1 0x4c
+#define AVS_MBOX_FREQUENCY 0x50
+
+/* AVS Commands */
+#define AVS_CMD_AVAILABLE 0x00
+#define AVS_CMD_DISABLE 0x10
+#define AVS_CMD_ENABLE 0x11
+#define AVS_CMD_S2_ENTER 0x12
+#define AVS_CMD_S2_EXIT 0x13
+#define AVS_CMD_BBM_ENTER 0x14
+#define AVS_CMD_BBM_EXIT 0x15
+#define AVS_CMD_S3_ENTER 0x16
+#define AVS_CMD_S3_EXIT 0x17
+#define AVS_CMD_BALANCE 0x18
+/* PMAP and P-STATE commands */
+#define AVS_CMD_GET_PMAP 0x30
+#define AVS_CMD_SET_PMAP 0x31
+#define AVS_CMD_GET_PSTATE 0x40
+#define AVS_CMD_SET_PSTATE 0x41
+
+/* Different modes AVS supports (for GET_PMAP/SET_PMAP) */
+#define AVS_MODE_AVS 0x0
+#define AVS_MODE_DFS 0x1
+#define AVS_MODE_DVS 0x2
+#define AVS_MODE_DVFS 0x3
+
+/*
+ * PMAP parameter p1
+ * unused:31-24, mdiv_p0:23-16, unused:15-14, pdiv:13-10 , ndiv_int:9-0
+ */
+#define NDIV_INT_SHIFT 0
+#define NDIV_INT_MASK 0x3ff
+#define PDIV_SHIFT 10
+#define PDIV_MASK 0xf
+#define MDIV_P0_SHIFT 16
+#define MDIV_P0_MASK 0xff
+/*
+ * PMAP parameter p2
+ * mdiv_p4:31-24, mdiv_p3:23-16, mdiv_p2:15:8, mdiv_p1:7:0
+ */
+#define MDIV_P1_SHIFT 0
+#define MDIV_P1_MASK 0xff
+#define MDIV_P2_SHIFT 8
+#define MDIV_P2_MASK 0xff
+#define MDIV_P3_SHIFT 16
+#define MDIV_P3_MASK 0xff
+#define MDIV_P4_SHIFT 24
+#define MDIV_P4_MASK 0xff
+
+/* Different P-STATES AVS supports (for GET_PSTATE/SET_PSTATE) */
+#define AVS_PSTATE_P0 0x0
+#define AVS_PSTATE_P1 0x1
+#define AVS_PSTATE_P2 0x2
+#define AVS_PSTATE_P3 0x3
+#define AVS_PSTATE_P4 0x4
+#define AVS_PSTATE_MAX AVS_PSTATE_P4
+
+/* CPU L2 Interrupt Controller Registers */
+#define AVS_CPU_L2_SET0 0x04
+#define AVS_CPU_L2_INT_MASK BIT(31)
+
+/* AVS Command Status Values */
+#define AVS_STATUS_CLEAR 0x00
+/* Command/notification accepted */
+#define AVS_STATUS_SUCCESS 0xf0
+/* Command/notification rejected */
+#define AVS_STATUS_FAILURE 0xff
+/* Invalid command/notification (unknown) */
+#define AVS_STATUS_INVALID 0xf1
+/* Non-AVS modes are not supported */
+#define AVS_STATUS_NO_SUPP 0xf2
+/* Cannot set P-State until P-Map supplied */
+#define AVS_STATUS_NO_MAP 0xf3
+/* Cannot change P-Map after initial P-Map set */
+#define AVS_STATUS_MAP_SET 0xf4
+/* Max AVS status; higher numbers are used for debugging */
+#define AVS_STATUS_MAX 0xff
+
+/* Other AVS related constants */
+#define AVS_LOOP_LIMIT 10000
+#define AVS_TIMEOUT 300 /* in ms; expected completion is < 10ms */
+#define AVS_FIRMWARE_MAGIC 0xa11600d1
+
+#define BRCM_AVS_CPUFREQ_PREFIX "brcmstb-avs"
+#define BRCM_AVS_CPUFREQ_NAME BRCM_AVS_CPUFREQ_PREFIX "-cpufreq"
+#define BRCM_AVS_CPU_DATA "brcm,avs-cpu-data-mem"
+#define BRCM_AVS_CPU_INTR "brcm,avs-cpu-l2-intr"
+#define BRCM_AVS_HOST_INTR "sw_intr"
+
+struct pmap {
+ unsigned int mode;
+ unsigned int p1;
+ unsigned int p2;
+ unsigned int state;
+};
+
+struct private_data {
+ void __iomem *base;
+ void __iomem *avs_intr_base;
+ struct device *dev;
+ struct completion done;
+ struct semaphore sem;
+ struct pmap pmap;
+ int host_irq;
+};
+
+static void __iomem *__map_region(const char *name)
+{
+ struct device_node *np;
+ void __iomem *ptr;
+
+ np = of_find_compatible_node(NULL, NULL, name);
+ if (!np)
+ return NULL;
+
+ ptr = of_iomap(np, 0);
+ of_node_put(np);
+
+ return ptr;
+}
+
+static unsigned long wait_for_avs_command(struct private_data *priv,
+ unsigned long timeout)
+{
+ unsigned long time_left = 0;
+ u32 val;
+
+ /* Event driven, wait for the command interrupt */
+ if (priv->host_irq >= 0)
+ return wait_for_completion_timeout(&priv->done,
+ msecs_to_jiffies(timeout));
+
+ /* Polling for command completion */
+ do {
+ time_left = timeout;
+ val = readl(priv->base + AVS_MBOX_STATUS);
+ if (val)
+ break;
+
+ usleep_range(1000, 2000);
+ } while (--timeout);
+
+ return time_left;
+}
+
+static int __issue_avs_command(struct private_data *priv, unsigned int cmd,
+ unsigned int num_in, unsigned int num_out,
+ u32 args[])
+{
+ void __iomem *base = priv->base;
+ unsigned long time_left;
+ unsigned int i;
+ int ret;
+ u32 val;
+
+ ret = down_interruptible(&priv->sem);
+ if (ret)
+ return ret;
+
+ /*
+ * Make sure no other command is currently running: cmd is 0 if AVS
+ * co-processor is idle. Due to the guard above, we should almost never
+ * have to wait here.
+ */
+ for (i = 0, val = 1; val != 0 && i < AVS_LOOP_LIMIT; i++)
+ val = readl(base + AVS_MBOX_COMMAND);
+
+ /* Give the caller a chance to retry if AVS is busy. */
+ if (i == AVS_LOOP_LIMIT) {
+ ret = -EAGAIN;
+ goto out;
+ }
+
+ /* Clear status before we begin. */
+ writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS);
+
+ /* Provide input parameters */
+ for (i = 0; i < num_in; i++)
+ writel(args[i], base + AVS_MBOX_PARAM(i));
+
+ /* Protect from spurious interrupts. */
+ reinit_completion(&priv->done);
+
+ /* Now issue the command & tell firmware to wake up to process it. */
+ writel(cmd, base + AVS_MBOX_COMMAND);
+ writel(AVS_CPU_L2_INT_MASK, priv->avs_intr_base + AVS_CPU_L2_SET0);
+
+ /* Wait for AVS co-processor to finish processing the command. */
+ time_left = wait_for_avs_command(priv, AVS_TIMEOUT);
+
+ /*
+ * If the AVS status is not in the expected range, it means AVS didn't
+ * complete our command in time, and we return an error. Also, if there
+ * is no "time left", we timed out waiting for the interrupt.
+ */
+ val = readl(base + AVS_MBOX_STATUS);
+ if (time_left == 0 || val == 0 || val > AVS_STATUS_MAX) {
+ dev_err(priv->dev, "AVS command %#x didn't complete in time\n",
+ cmd);
+ dev_err(priv->dev, " Time left: %u ms, AVS status: %#x\n",
+ jiffies_to_msecs(time_left), val);
+ ret = -ETIMEDOUT;
+ goto out;
+ }
+
+ /* Process returned values */
+ for (i = 0; i < num_out; i++)
+ args[i] = readl(base + AVS_MBOX_PARAM(i));
+
+ /* Clear status to tell AVS co-processor we are done. */
+ writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS);
+
+ /* Convert firmware errors to errno's as much as possible. */
+ switch (val) {
+ case AVS_STATUS_INVALID:
+ ret = -EINVAL;
+ break;
+ case AVS_STATUS_NO_SUPP:
+ ret = -ENOTSUPP;
+ break;
+ case AVS_STATUS_NO_MAP:
+ ret = -ENOENT;
+ break;
+ case AVS_STATUS_MAP_SET:
+ ret = -EEXIST;
+ break;
+ case AVS_STATUS_FAILURE:
+ ret = -EIO;
+ break;
+ }
+
+out:
+ up(&priv->sem);
+
+ return ret;
+}
+
+static irqreturn_t irq_handler(int irq, void *data)
+{
+ struct private_data *priv = data;
+
+ /* AVS command completed execution. Wake up __issue_avs_command(). */
+ complete(&priv->done);
+
+ return IRQ_HANDLED;
+}
+
+static char *brcm_avs_mode_to_string(unsigned int mode)
+{
+ switch (mode) {
+ case AVS_MODE_AVS:
+ return "AVS";
+ case AVS_MODE_DFS:
+ return "DFS";
+ case AVS_MODE_DVS:
+ return "DVS";
+ case AVS_MODE_DVFS:
+ return "DVFS";
+ }
+ return NULL;
+}
+
+static void brcm_avs_parse_p1(u32 p1, unsigned int *mdiv_p0, unsigned int *pdiv,
+ unsigned int *ndiv)
+{
+ *mdiv_p0 = (p1 >> MDIV_P0_SHIFT) & MDIV_P0_MASK;
+ *pdiv = (p1 >> PDIV_SHIFT) & PDIV_MASK;
+ *ndiv = (p1 >> NDIV_INT_SHIFT) & NDIV_INT_MASK;
+}
+
+static void brcm_avs_parse_p2(u32 p2, unsigned int *mdiv_p1,
+ unsigned int *mdiv_p2, unsigned int *mdiv_p3,
+ unsigned int *mdiv_p4)
+{
+ *mdiv_p4 = (p2 >> MDIV_P4_SHIFT) & MDIV_P4_MASK;
+ *mdiv_p3 = (p2 >> MDIV_P3_SHIFT) & MDIV_P3_MASK;
+ *mdiv_p2 = (p2 >> MDIV_P2_SHIFT) & MDIV_P2_MASK;
+ *mdiv_p1 = (p2 >> MDIV_P1_SHIFT) & MDIV_P1_MASK;
+}
+
+static int brcm_avs_get_pmap(struct private_data *priv, struct pmap *pmap)
+{
+ u32 args[AVS_MAX_CMD_ARGS];
+ int ret;
+
+ ret = __issue_avs_command(priv, AVS_CMD_GET_PMAP, 0, 4, args);
+ if (ret || !pmap)
+ return ret;
+
+ pmap->mode = args[0];
+ pmap->p1 = args[1];
+ pmap->p2 = args[2];
+ pmap->state = args[3];
+
+ return 0;
+}
+
+static int brcm_avs_set_pmap(struct private_data *priv, struct pmap *pmap)
+{
+ u32 args[AVS_MAX_CMD_ARGS];
+
+ args[0] = pmap->mode;
+ args[1] = pmap->p1;
+ args[2] = pmap->p2;
+ args[3] = pmap->state;
+
+ return __issue_avs_command(priv, AVS_CMD_SET_PMAP, 4, 0, args);
+}
+
+static int brcm_avs_get_pstate(struct private_data *priv, unsigned int *pstate)
+{
+ u32 args[AVS_MAX_CMD_ARGS];
+ int ret;
+
+ ret = __issue_avs_command(priv, AVS_CMD_GET_PSTATE, 0, 1, args);
+ if (ret)
+ return ret;
+ *pstate = args[0];
+
+ return 0;
+}
+
+static int brcm_avs_set_pstate(struct private_data *priv, unsigned int pstate)
+{
+ u32 args[AVS_MAX_CMD_ARGS];
+
+ args[0] = pstate;
+
+ return __issue_avs_command(priv, AVS_CMD_SET_PSTATE, 1, 0, args);
+
+}
+
+static u32 brcm_avs_get_voltage(void __iomem *base)
+{
+ return readl(base + AVS_MBOX_VOLTAGE1);
+}
+
+static u32 brcm_avs_get_frequency(void __iomem *base)
+{
+ return readl(base + AVS_MBOX_FREQUENCY) * 1000; /* in kHz */
+}
+
+/*
+ * We determine which frequencies are supported by cycling through all P-states
+ * and reading back what frequency we are running at for each P-state.
+ */
+static struct cpufreq_frequency_table *
+brcm_avs_get_freq_table(struct device *dev, struct private_data *priv)
+{
+ struct cpufreq_frequency_table *table;
+ unsigned int pstate;
+ int i, ret;
+
+ /* Remember P-state for later */
+ ret = brcm_avs_get_pstate(priv, &pstate);
+ if (ret)
+ return ERR_PTR(ret);
+
+ /*
+ * We allocate space for the 5 different P-STATES AVS,
+ * plus extra space for a terminating element.
+ */
+ table = devm_kcalloc(dev, AVS_PSTATE_MAX + 1 + 1, sizeof(*table),
+ GFP_KERNEL);
+ if (!table)
+ return ERR_PTR(-ENOMEM);
+
+ for (i = AVS_PSTATE_P0; i <= AVS_PSTATE_MAX; i++) {
+ ret = brcm_avs_set_pstate(priv, i);
+ if (ret)
+ return ERR_PTR(ret);
+ table[i].frequency = brcm_avs_get_frequency(priv->base);
+ table[i].driver_data = i;
+ }
+ table[i].frequency = CPUFREQ_TABLE_END;
+
+ /* Restore P-state */
+ ret = brcm_avs_set_pstate(priv, pstate);
+ if (ret)
+ return ERR_PTR(ret);
+
+ return table;
+}
+
+/*
+ * To ensure the right firmware is running we need to
+ * - check the MAGIC matches what we expect
+ * - brcm_avs_get_pmap() doesn't return -ENOTSUPP or -EINVAL
+ * We need to set up our interrupt handling before calling brcm_avs_get_pmap()!
+ */
+static bool brcm_avs_is_firmware_loaded(struct private_data *priv)
+{
+ u32 magic;
+ int rc;
+
+ rc = brcm_avs_get_pmap(priv, NULL);
+ magic = readl(priv->base + AVS_MBOX_MAGIC);
+
+ return (magic == AVS_FIRMWARE_MAGIC) && ((rc != -ENOTSUPP) ||
+ (rc != -EINVAL));
+}
+
+static unsigned int brcm_avs_cpufreq_get(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+ struct private_data *priv = policy->driver_data;
+
+ cpufreq_cpu_put(policy);
+
+ return brcm_avs_get_frequency(priv->base);
+}
+
+static int brcm_avs_target_index(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ return brcm_avs_set_pstate(policy->driver_data,
+ policy->freq_table[index].driver_data);
+}
+
+static int brcm_avs_suspend(struct cpufreq_policy *policy)
+{
+ struct private_data *priv = policy->driver_data;
+ int ret;
+
+ ret = brcm_avs_get_pmap(priv, &priv->pmap);
+ if (ret)
+ return ret;
+
+ /*
+ * We can't use the P-state returned by brcm_avs_get_pmap(), since
+ * that's the initial P-state from when the P-map was downloaded to the
+ * AVS co-processor, not necessarily the P-state we are running at now.
+ * So, we get the current P-state explicitly.
+ */
+ ret = brcm_avs_get_pstate(priv, &priv->pmap.state);
+ if (ret)
+ return ret;
+
+ /* This is best effort. Nothing to do if it fails. */
+ (void)__issue_avs_command(priv, AVS_CMD_S2_ENTER, 0, 0, NULL);
+
+ return 0;
+}
+
+static int brcm_avs_resume(struct cpufreq_policy *policy)
+{
+ struct private_data *priv = policy->driver_data;
+ int ret;
+
+ /* This is best effort. Nothing to do if it fails. */
+ (void)__issue_avs_command(priv, AVS_CMD_S2_EXIT, 0, 0, NULL);
+
+ ret = brcm_avs_set_pmap(priv, &priv->pmap);
+ if (ret == -EEXIST) {
+ struct platform_device *pdev = cpufreq_get_driver_data();
+ struct device *dev = &pdev->dev;
+
+ dev_warn(dev, "PMAP was already set\n");
+ ret = 0;
+ }
+
+ return ret;
+}
+
+/*
+ * All initialization code that we only want to execute once goes here. Setup
+ * code that can be re-tried on every core (if it failed before) can go into
+ * brcm_avs_cpufreq_init().
+ */
+static int brcm_avs_prepare_init(struct platform_device *pdev)
+{
+ struct private_data *priv;
+ struct device *dev;
+ int ret;
+
+ dev = &pdev->dev;
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->dev = dev;
+ sema_init(&priv->sem, 1);
+ init_completion(&priv->done);
+ platform_set_drvdata(pdev, priv);
+
+ priv->base = __map_region(BRCM_AVS_CPU_DATA);
+ if (!priv->base) {
+ dev_err(dev, "Couldn't find property %s in device tree.\n",
+ BRCM_AVS_CPU_DATA);
+ return -ENOENT;
+ }
+
+ priv->avs_intr_base = __map_region(BRCM_AVS_CPU_INTR);
+ if (!priv->avs_intr_base) {
+ dev_err(dev, "Couldn't find property %s in device tree.\n",
+ BRCM_AVS_CPU_INTR);
+ ret = -ENOENT;
+ goto unmap_base;
+ }
+
+ priv->host_irq = platform_get_irq_byname(pdev, BRCM_AVS_HOST_INTR);
+
+ ret = devm_request_irq(dev, priv->host_irq, irq_handler,
+ IRQF_TRIGGER_RISING,
+ BRCM_AVS_HOST_INTR, priv);
+ if (ret && priv->host_irq >= 0) {
+ dev_err(dev, "IRQ request failed: %s (%d) -- %d\n",
+ BRCM_AVS_HOST_INTR, priv->host_irq, ret);
+ goto unmap_intr_base;
+ }
+
+ if (brcm_avs_is_firmware_loaded(priv))
+ return 0;
+
+ dev_err(dev, "AVS firmware is not loaded or doesn't support DVFS\n");
+ ret = -ENODEV;
+
+unmap_intr_base:
+ iounmap(priv->avs_intr_base);
+unmap_base:
+ iounmap(priv->base);
+
+ return ret;
+}
+
+static void brcm_avs_prepare_uninit(struct platform_device *pdev)
+{
+ struct private_data *priv;
+
+ priv = platform_get_drvdata(pdev);
+
+ iounmap(priv->avs_intr_base);
+ iounmap(priv->base);
+}
+
+static int brcm_avs_cpufreq_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *freq_table;
+ struct platform_device *pdev;
+ struct private_data *priv;
+ struct device *dev;
+ int ret;
+
+ pdev = cpufreq_get_driver_data();
+ priv = platform_get_drvdata(pdev);
+ policy->driver_data = priv;
+ dev = &pdev->dev;
+
+ freq_table = brcm_avs_get_freq_table(dev, priv);
+ if (IS_ERR(freq_table)) {
+ ret = PTR_ERR(freq_table);
+ dev_err(dev, "Couldn't determine frequency table (%d).\n", ret);
+ return ret;
+ }
+
+ policy->freq_table = freq_table;
+
+ /* All cores share the same clock and thus the same policy. */
+ cpumask_setall(policy->cpus);
+
+ ret = __issue_avs_command(priv, AVS_CMD_ENABLE, 0, 0, NULL);
+ if (!ret) {
+ unsigned int pstate;
+
+ ret = brcm_avs_get_pstate(priv, &pstate);
+ if (!ret) {
+ policy->cur = freq_table[pstate].frequency;
+ dev_info(dev, "registered\n");
+ return 0;
+ }
+ }
+
+ dev_err(dev, "couldn't initialize driver (%d)\n", ret);
+
+ return ret;
+}
+
+static ssize_t show_brcm_avs_pstate(struct cpufreq_policy *policy, char *buf)
+{
+ struct private_data *priv = policy->driver_data;
+ unsigned int pstate;
+
+ if (brcm_avs_get_pstate(priv, &pstate))
+ return sprintf(buf, "<unknown>\n");
+
+ return sprintf(buf, "%u\n", pstate);
+}
+
+static ssize_t show_brcm_avs_mode(struct cpufreq_policy *policy, char *buf)
+{
+ struct private_data *priv = policy->driver_data;
+ struct pmap pmap;
+
+ if (brcm_avs_get_pmap(priv, &pmap))
+ return sprintf(buf, "<unknown>\n");
+
+ return sprintf(buf, "%s %u\n", brcm_avs_mode_to_string(pmap.mode),
+ pmap.mode);
+}
+
+static ssize_t show_brcm_avs_pmap(struct cpufreq_policy *policy, char *buf)
+{
+ unsigned int mdiv_p0, mdiv_p1, mdiv_p2, mdiv_p3, mdiv_p4;
+ struct private_data *priv = policy->driver_data;
+ unsigned int ndiv, pdiv;
+ struct pmap pmap;
+
+ if (brcm_avs_get_pmap(priv, &pmap))
+ return sprintf(buf, "<unknown>\n");
+
+ brcm_avs_parse_p1(pmap.p1, &mdiv_p0, &pdiv, &ndiv);
+ brcm_avs_parse_p2(pmap.p2, &mdiv_p1, &mdiv_p2, &mdiv_p3, &mdiv_p4);
+
+ return sprintf(buf, "0x%08x 0x%08x %u %u %u %u %u %u %u %u %u\n",
+ pmap.p1, pmap.p2, ndiv, pdiv, mdiv_p0, mdiv_p1, mdiv_p2,
+ mdiv_p3, mdiv_p4, pmap.mode, pmap.state);
+}
+
+static ssize_t show_brcm_avs_voltage(struct cpufreq_policy *policy, char *buf)
+{
+ struct private_data *priv = policy->driver_data;
+
+ return sprintf(buf, "0x%08x\n", brcm_avs_get_voltage(priv->base));
+}
+
+static ssize_t show_brcm_avs_frequency(struct cpufreq_policy *policy, char *buf)
+{
+ struct private_data *priv = policy->driver_data;
+
+ return sprintf(buf, "0x%08x\n", brcm_avs_get_frequency(priv->base));
+}
+
+cpufreq_freq_attr_ro(brcm_avs_pstate);
+cpufreq_freq_attr_ro(brcm_avs_mode);
+cpufreq_freq_attr_ro(brcm_avs_pmap);
+cpufreq_freq_attr_ro(brcm_avs_voltage);
+cpufreq_freq_attr_ro(brcm_avs_frequency);
+
+static struct freq_attr *brcm_avs_cpufreq_attr[] = {
+ &cpufreq_freq_attr_scaling_available_freqs,
+ &brcm_avs_pstate,
+ &brcm_avs_mode,
+ &brcm_avs_pmap,
+ &brcm_avs_voltage,
+ &brcm_avs_frequency,
+ NULL
+};
+
+static struct cpufreq_driver brcm_avs_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = brcm_avs_target_index,
+ .get = brcm_avs_cpufreq_get,
+ .suspend = brcm_avs_suspend,
+ .resume = brcm_avs_resume,
+ .init = brcm_avs_cpufreq_init,
+ .attr = brcm_avs_cpufreq_attr,
+ .name = BRCM_AVS_CPUFREQ_PREFIX,
+};
+
+static int brcm_avs_cpufreq_probe(struct platform_device *pdev)
+{
+ int ret;
+
+ ret = brcm_avs_prepare_init(pdev);
+ if (ret)
+ return ret;
+
+ brcm_avs_driver.driver_data = pdev;
+
+ ret = cpufreq_register_driver(&brcm_avs_driver);
+ if (ret)
+ brcm_avs_prepare_uninit(pdev);
+
+ return ret;
+}
+
+static void brcm_avs_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&brcm_avs_driver);
+
+ brcm_avs_prepare_uninit(pdev);
+}
+
+static const struct of_device_id brcm_avs_cpufreq_match[] = {
+ { .compatible = BRCM_AVS_CPU_DATA },
+ { }
+};
+MODULE_DEVICE_TABLE(of, brcm_avs_cpufreq_match);
+
+static struct platform_driver brcm_avs_cpufreq_platdrv = {
+ .driver = {
+ .name = BRCM_AVS_CPUFREQ_NAME,
+ .of_match_table = brcm_avs_cpufreq_match,
+ },
+ .probe = brcm_avs_cpufreq_probe,
+ .remove_new = brcm_avs_cpufreq_remove,
+};
+module_platform_driver(brcm_avs_cpufreq_platdrv);
+
+MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>");
+MODULE_DESCRIPTION("CPUfreq driver for Broadcom STB AVS");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
new file mode 100644
index 0000000000..fe08ca419b
--- /dev/null
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -0,0 +1,1017 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * CPPC (Collaborative Processor Performance Control) driver for
+ * interfacing with the CPUfreq layer and governors. See
+ * cppc_acpi.c for CPPC specific methods.
+ *
+ * (C) Copyright 2014, 2015 Linaro Ltd.
+ * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org>
+ */
+
+#define pr_fmt(fmt) "CPPC Cpufreq:" fmt
+
+#include <linux/arch_topology.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/dmi.h>
+#include <linux/irq_work.h>
+#include <linux/kthread.h>
+#include <linux/time.h>
+#include <linux/vmalloc.h>
+#include <uapi/linux/sched/types.h>
+
+#include <asm/unaligned.h>
+
+#include <acpi/cppc_acpi.h>
+
+/* Minimum struct length needed for the DMI processor entry we want */
+#define DMI_ENTRY_PROCESSOR_MIN_LENGTH 48
+
+/* Offset in the DMI processor structure for the max frequency */
+#define DMI_PROCESSOR_MAX_SPEED 0x14
+
+/*
+ * This list contains information parsed from per CPU ACPI _CPC and _PSD
+ * structures: e.g. the highest and lowest supported performance, capabilities,
+ * desired performance, level requested etc. Depending on the share_type, not
+ * all CPUs will have an entry in the list.
+ */
+static LIST_HEAD(cpu_data_list);
+
+static bool boost_supported;
+
+struct cppc_workaround_oem_info {
+ char oem_id[ACPI_OEM_ID_SIZE + 1];
+ char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
+ u32 oem_revision;
+};
+
+static struct cppc_workaround_oem_info wa_info[] = {
+ {
+ .oem_id = "HISI ",
+ .oem_table_id = "HIP07 ",
+ .oem_revision = 0,
+ }, {
+ .oem_id = "HISI ",
+ .oem_table_id = "HIP08 ",
+ .oem_revision = 0,
+ }
+};
+
+static struct cpufreq_driver cppc_cpufreq_driver;
+
+static enum {
+ FIE_UNSET = -1,
+ FIE_ENABLED,
+ FIE_DISABLED
+} fie_disabled = FIE_UNSET;
+
+#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE
+module_param(fie_disabled, int, 0444);
+MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");
+
+/* Frequency invariance support */
+struct cppc_freq_invariance {
+ int cpu;
+ struct irq_work irq_work;
+ struct kthread_work work;
+ struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
+ struct cppc_cpudata *cpu_data;
+};
+
+static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
+static struct kthread_worker *kworker_fie;
+
+static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu);
+static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
+ struct cppc_perf_fb_ctrs *fb_ctrs_t0,
+ struct cppc_perf_fb_ctrs *fb_ctrs_t1);
+
+/**
+ * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance
+ * @work: The work item.
+ *
+ * The CPPC driver register itself with the topology core to provide its own
+ * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which
+ * gets called by the scheduler on every tick.
+ *
+ * Note that the arch specific counters have higher priority than CPPC counters,
+ * if available, though the CPPC driver doesn't need to have any special
+ * handling for that.
+ *
+ * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we
+ * reach here from hard-irq context), which then schedules a normal work item
+ * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable
+ * based on the counter updates since the last tick.
+ */
+static void cppc_scale_freq_workfn(struct kthread_work *work)
+{
+ struct cppc_freq_invariance *cppc_fi;
+ struct cppc_perf_fb_ctrs fb_ctrs = {0};
+ struct cppc_cpudata *cpu_data;
+ unsigned long local_freq_scale;
+ u64 perf;
+
+ cppc_fi = container_of(work, struct cppc_freq_invariance, work);
+ cpu_data = cppc_fi->cpu_data;
+
+ if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) {
+ pr_warn("%s: failed to read perf counters\n", __func__);
+ return;
+ }
+
+ perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs,
+ &fb_ctrs);
+ cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
+
+ perf <<= SCHED_CAPACITY_SHIFT;
+ local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);
+
+ /* This can happen due to counter's overflow */
+ if (unlikely(local_freq_scale > 1024))
+ local_freq_scale = 1024;
+
+ per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;
+}
+
+static void cppc_irq_work(struct irq_work *irq_work)
+{
+ struct cppc_freq_invariance *cppc_fi;
+
+ cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
+ kthread_queue_work(kworker_fie, &cppc_fi->work);
+}
+
+static void cppc_scale_freq_tick(void)
+{
+ struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id());
+
+ /*
+ * cppc_get_perf_ctrs() can potentially sleep, call that from the right
+ * context.
+ */
+ irq_work_queue(&cppc_fi->irq_work);
+}
+
+static struct scale_freq_data cppc_sftd = {
+ .source = SCALE_FREQ_SOURCE_CPPC,
+ .set_freq_scale = cppc_scale_freq_tick,
+};
+
+static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
+{
+ struct cppc_freq_invariance *cppc_fi;
+ int cpu, ret;
+
+ if (fie_disabled)
+ return;
+
+ for_each_cpu(cpu, policy->cpus) {
+ cppc_fi = &per_cpu(cppc_freq_inv, cpu);
+ cppc_fi->cpu = cpu;
+ cppc_fi->cpu_data = policy->driver_data;
+ kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);
+ init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
+
+ ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);
+ if (ret) {
+ pr_warn("%s: failed to read perf counters for cpu:%d: %d\n",
+ __func__, cpu, ret);
+
+ /*
+ * Don't abort if the CPU was offline while the driver
+ * was getting registered.
+ */
+ if (cpu_online(cpu))
+ return;
+ }
+ }
+
+ /* Register for freq-invariance */
+ topology_set_scale_freq_source(&cppc_sftd, policy->cpus);
+}
+
+/*
+ * We free all the resources on policy's removal and not on CPU removal as the
+ * irq-work are per-cpu and the hotplug core takes care of flushing the pending
+ * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work
+ * fires on another CPU after the concerned CPU is removed, it won't harm.
+ *
+ * We just need to make sure to remove them all on policy->exit().
+ */
+static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
+{
+ struct cppc_freq_invariance *cppc_fi;
+ int cpu;
+
+ if (fie_disabled)
+ return;
+
+ /* policy->cpus will be empty here, use related_cpus instead */
+ topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);
+
+ for_each_cpu(cpu, policy->related_cpus) {
+ cppc_fi = &per_cpu(cppc_freq_inv, cpu);
+ irq_work_sync(&cppc_fi->irq_work);
+ kthread_cancel_work_sync(&cppc_fi->work);
+ }
+}
+
+static void __init cppc_freq_invariance_init(void)
+{
+ struct sched_attr attr = {
+ .size = sizeof(struct sched_attr),
+ .sched_policy = SCHED_DEADLINE,
+ .sched_nice = 0,
+ .sched_priority = 0,
+ /*
+ * Fake (unused) bandwidth; workaround to "fix"
+ * priority inheritance.
+ */
+ .sched_runtime = 1000000,
+ .sched_deadline = 10000000,
+ .sched_period = 10000000,
+ };
+ int ret;
+
+ if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) {
+ fie_disabled = FIE_ENABLED;
+ if (cppc_perf_ctrs_in_pcc()) {
+ pr_info("FIE not enabled on systems with registers in PCC\n");
+ fie_disabled = FIE_DISABLED;
+ }
+ }
+
+ if (fie_disabled)
+ return;
+
+ kworker_fie = kthread_create_worker(0, "cppc_fie");
+ if (IS_ERR(kworker_fie)) {
+ pr_warn("%s: failed to create kworker_fie: %ld\n", __func__,
+ PTR_ERR(kworker_fie));
+ fie_disabled = FIE_DISABLED;
+ return;
+ }
+
+ ret = sched_setattr_nocheck(kworker_fie->task, &attr);
+ if (ret) {
+ pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__,
+ ret);
+ kthread_destroy_worker(kworker_fie);
+ fie_disabled = FIE_DISABLED;
+ }
+}
+
+static void cppc_freq_invariance_exit(void)
+{
+ if (fie_disabled)
+ return;
+
+ kthread_destroy_worker(kworker_fie);
+}
+
+#else
+static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
+{
+}
+
+static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
+{
+}
+
+static inline void cppc_freq_invariance_init(void)
+{
+}
+
+static inline void cppc_freq_invariance_exit(void)
+{
+}
+#endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */
+
+/* Callback function used to retrieve the max frequency from DMI */
+static void cppc_find_dmi_mhz(const struct dmi_header *dm, void *private)
+{
+ const u8 *dmi_data = (const u8 *)dm;
+ u16 *mhz = (u16 *)private;
+
+ if (dm->type == DMI_ENTRY_PROCESSOR &&
+ dm->length >= DMI_ENTRY_PROCESSOR_MIN_LENGTH) {
+ u16 val = (u16)get_unaligned((const u16 *)
+ (dmi_data + DMI_PROCESSOR_MAX_SPEED));
+ *mhz = val > *mhz ? val : *mhz;
+ }
+}
+
+/* Look up the max frequency in DMI */
+static u64 cppc_get_dmi_max_khz(void)
+{
+ u16 mhz = 0;
+
+ dmi_walk(cppc_find_dmi_mhz, &mhz);
+
+ /*
+ * Real stupid fallback value, just in case there is no
+ * actual value set.
+ */
+ mhz = mhz ? mhz : 1;
+
+ return (1000 * mhz);
+}
+
+/*
+ * If CPPC lowest_freq and nominal_freq registers are exposed then we can
+ * use them to convert perf to freq and vice versa. The conversion is
+ * extrapolated as an affine function passing by the 2 points:
+ * - (Low perf, Low freq)
+ * - (Nominal perf, Nominal perf)
+ */
+static unsigned int cppc_cpufreq_perf_to_khz(struct cppc_cpudata *cpu_data,
+ unsigned int perf)
+{
+ struct cppc_perf_caps *caps = &cpu_data->perf_caps;
+ s64 retval, offset = 0;
+ static u64 max_khz;
+ u64 mul, div;
+
+ if (caps->lowest_freq && caps->nominal_freq) {
+ mul = caps->nominal_freq - caps->lowest_freq;
+ div = caps->nominal_perf - caps->lowest_perf;
+ offset = caps->nominal_freq - div64_u64(caps->nominal_perf * mul, div);
+ } else {
+ if (!max_khz)
+ max_khz = cppc_get_dmi_max_khz();
+ mul = max_khz;
+ div = caps->highest_perf;
+ }
+
+ retval = offset + div64_u64(perf * mul, div);
+ if (retval >= 0)
+ return retval;
+ return 0;
+}
+
+static unsigned int cppc_cpufreq_khz_to_perf(struct cppc_cpudata *cpu_data,
+ unsigned int freq)
+{
+ struct cppc_perf_caps *caps = &cpu_data->perf_caps;
+ s64 retval, offset = 0;
+ static u64 max_khz;
+ u64 mul, div;
+
+ if (caps->lowest_freq && caps->nominal_freq) {
+ mul = caps->nominal_perf - caps->lowest_perf;
+ div = caps->nominal_freq - caps->lowest_freq;
+ offset = caps->nominal_perf - div64_u64(caps->nominal_freq * mul, div);
+ } else {
+ if (!max_khz)
+ max_khz = cppc_get_dmi_max_khz();
+ mul = caps->highest_perf;
+ div = max_khz;
+ }
+
+ retval = offset + div64_u64(freq * mul, div);
+ if (retval >= 0)
+ return retval;
+ return 0;
+}
+
+static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+ unsigned int cpu = policy->cpu;
+ struct cpufreq_freqs freqs;
+ u32 desired_perf;
+ int ret = 0;
+
+ desired_perf = cppc_cpufreq_khz_to_perf(cpu_data, target_freq);
+ /* Return if it is exactly the same perf */
+ if (desired_perf == cpu_data->perf_ctrls.desired_perf)
+ return ret;
+
+ cpu_data->perf_ctrls.desired_perf = desired_perf;
+ freqs.old = policy->cur;
+ freqs.new = target_freq;
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+ ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
+ cpufreq_freq_transition_end(policy, &freqs, ret != 0);
+
+ if (ret)
+ pr_debug("Failed to set target on CPU:%d. ret:%d\n",
+ cpu, ret);
+
+ return ret;
+}
+
+static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+ unsigned int cpu = policy->cpu;
+ u32 desired_perf;
+ int ret;
+
+ desired_perf = cppc_cpufreq_khz_to_perf(cpu_data, target_freq);
+ cpu_data->perf_ctrls.desired_perf = desired_perf;
+ ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
+
+ if (ret) {
+ pr_debug("Failed to set target on CPU:%d. ret:%d\n",
+ cpu, ret);
+ return 0;
+ }
+
+ return target_freq;
+}
+
+static int cppc_verify_policy(struct cpufreq_policy_data *policy)
+{
+ cpufreq_verify_within_cpu_limits(policy);
+ return 0;
+}
+
+/*
+ * The PCC subspace describes the rate at which platform can accept commands
+ * on the shared PCC channel (including READs which do not count towards freq
+ * transition requests), so ideally we need to use the PCC values as a fallback
+ * if we don't have a platform specific transition_delay_us
+ */
+#ifdef CONFIG_ARM64
+#include <asm/cputype.h>
+
+static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
+{
+ unsigned long implementor = read_cpuid_implementor();
+ unsigned long part_num = read_cpuid_part_number();
+
+ switch (implementor) {
+ case ARM_CPU_IMP_QCOM:
+ switch (part_num) {
+ case QCOM_CPU_PART_FALKOR_V1:
+ case QCOM_CPU_PART_FALKOR:
+ return 10000;
+ }
+ }
+ return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;
+}
+#else
+static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
+{
+ return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;
+}
+#endif
+
+#if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL)
+
+static DEFINE_PER_CPU(unsigned int, efficiency_class);
+static void cppc_cpufreq_register_em(struct cpufreq_policy *policy);
+
+/* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */
+#define CPPC_EM_CAP_STEP (20)
+/* Increase the cost value by CPPC_EM_COST_STEP every performance state. */
+#define CPPC_EM_COST_STEP (1)
+/* Add a cost gap correspnding to the energy of 4 CPUs. */
+#define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \
+ / CPPC_EM_CAP_STEP)
+
+static unsigned int get_perf_level_count(struct cpufreq_policy *policy)
+{
+ struct cppc_perf_caps *perf_caps;
+ unsigned int min_cap, max_cap;
+ struct cppc_cpudata *cpu_data;
+ int cpu = policy->cpu;
+
+ cpu_data = policy->driver_data;
+ perf_caps = &cpu_data->perf_caps;
+ max_cap = arch_scale_cpu_capacity(cpu);
+ min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
+ perf_caps->highest_perf);
+ if ((min_cap == 0) || (max_cap < min_cap))
+ return 0;
+ return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP;
+}
+
+/*
+ * The cost is defined as:
+ * cost = power * max_frequency / frequency
+ */
+static inline unsigned long compute_cost(int cpu, int step)
+{
+ return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) +
+ step * CPPC_EM_COST_STEP;
+}
+
+static int cppc_get_cpu_power(struct device *cpu_dev,
+ unsigned long *power, unsigned long *KHz)
+{
+ unsigned long perf_step, perf_prev, perf, perf_check;
+ unsigned int min_step, max_step, step, step_check;
+ unsigned long prev_freq = *KHz;
+ unsigned int min_cap, max_cap;
+ struct cpufreq_policy *policy;
+
+ struct cppc_perf_caps *perf_caps;
+ struct cppc_cpudata *cpu_data;
+
+ policy = cpufreq_cpu_get_raw(cpu_dev->id);
+ cpu_data = policy->driver_data;
+ perf_caps = &cpu_data->perf_caps;
+ max_cap = arch_scale_cpu_capacity(cpu_dev->id);
+ min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
+ perf_caps->highest_perf);
+ perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf,
+ max_cap);
+ min_step = min_cap / CPPC_EM_CAP_STEP;
+ max_step = max_cap / CPPC_EM_CAP_STEP;
+
+ perf_prev = cppc_cpufreq_khz_to_perf(cpu_data, *KHz);
+ step = perf_prev / perf_step;
+
+ if (step > max_step)
+ return -EINVAL;
+
+ if (min_step == max_step) {
+ step = max_step;
+ perf = perf_caps->highest_perf;
+ } else if (step < min_step) {
+ step = min_step;
+ perf = perf_caps->lowest_perf;
+ } else {
+ step++;
+ if (step == max_step)
+ perf = perf_caps->highest_perf;
+ else
+ perf = step * perf_step;
+ }
+
+ *KHz = cppc_cpufreq_perf_to_khz(cpu_data, perf);
+ perf_check = cppc_cpufreq_khz_to_perf(cpu_data, *KHz);
+ step_check = perf_check / perf_step;
+
+ /*
+ * To avoid bad integer approximation, check that new frequency value
+ * increased and that the new frequency will be converted to the
+ * desired step value.
+ */
+ while ((*KHz == prev_freq) || (step_check != step)) {
+ perf++;
+ *KHz = cppc_cpufreq_perf_to_khz(cpu_data, perf);
+ perf_check = cppc_cpufreq_khz_to_perf(cpu_data, *KHz);
+ step_check = perf_check / perf_step;
+ }
+
+ /*
+ * With an artificial EM, only the cost value is used. Still the power
+ * is populated such as 0 < power < EM_MAX_POWER. This allows to add
+ * more sense to the artificial performance states.
+ */
+ *power = compute_cost(cpu_dev->id, step);
+
+ return 0;
+}
+
+static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz,
+ unsigned long *cost)
+{
+ unsigned long perf_step, perf_prev;
+ struct cppc_perf_caps *perf_caps;
+ struct cpufreq_policy *policy;
+ struct cppc_cpudata *cpu_data;
+ unsigned int max_cap;
+ int step;
+
+ policy = cpufreq_cpu_get_raw(cpu_dev->id);
+ cpu_data = policy->driver_data;
+ perf_caps = &cpu_data->perf_caps;
+ max_cap = arch_scale_cpu_capacity(cpu_dev->id);
+
+ perf_prev = cppc_cpufreq_khz_to_perf(cpu_data, KHz);
+ perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap;
+ step = perf_prev / perf_step;
+
+ *cost = compute_cost(cpu_dev->id, step);
+
+ return 0;
+}
+
+static int populate_efficiency_class(void)
+{
+ struct acpi_madt_generic_interrupt *gicc;
+ DECLARE_BITMAP(used_classes, 256) = {};
+ int class, cpu, index;
+
+ for_each_possible_cpu(cpu) {
+ gicc = acpi_cpu_get_madt_gicc(cpu);
+ class = gicc->efficiency_class;
+ bitmap_set(used_classes, class, 1);
+ }
+
+ if (bitmap_weight(used_classes, 256) <= 1) {
+ pr_debug("Efficiency classes are all equal (=%d). "
+ "No EM registered", class);
+ return -EINVAL;
+ }
+
+ /*
+ * Squeeze efficiency class values on [0:#efficiency_class-1].
+ * Values are per spec in [0:255].
+ */
+ index = 0;
+ for_each_set_bit(class, used_classes, 256) {
+ for_each_possible_cpu(cpu) {
+ gicc = acpi_cpu_get_madt_gicc(cpu);
+ if (gicc->efficiency_class == class)
+ per_cpu(efficiency_class, cpu) = index;
+ }
+ index++;
+ }
+ cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em;
+
+ return 0;
+}
+
+static void cppc_cpufreq_register_em(struct cpufreq_policy *policy)
+{
+ struct cppc_cpudata *cpu_data;
+ struct em_data_callback em_cb =
+ EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost);
+
+ cpu_data = policy->driver_data;
+ em_dev_register_perf_domain(get_cpu_device(policy->cpu),
+ get_perf_level_count(policy), &em_cb,
+ cpu_data->shared_cpu_map, 0);
+}
+
+#else
+static int populate_efficiency_class(void)
+{
+ return 0;
+}
+#endif
+
+static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu)
+{
+ struct cppc_cpudata *cpu_data;
+ int ret;
+
+ cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL);
+ if (!cpu_data)
+ goto out;
+
+ if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL))
+ goto free_cpu;
+
+ ret = acpi_get_psd_map(cpu, cpu_data);
+ if (ret) {
+ pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret);
+ goto free_mask;
+ }
+
+ ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps);
+ if (ret) {
+ pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret);
+ goto free_mask;
+ }
+
+ /* Convert the lowest and nominal freq from MHz to KHz */
+ cpu_data->perf_caps.lowest_freq *= 1000;
+ cpu_data->perf_caps.nominal_freq *= 1000;
+
+ list_add(&cpu_data->node, &cpu_data_list);
+
+ return cpu_data;
+
+free_mask:
+ free_cpumask_var(cpu_data->shared_cpu_map);
+free_cpu:
+ kfree(cpu_data);
+out:
+ return NULL;
+}
+
+static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+
+ list_del(&cpu_data->node);
+ free_cpumask_var(cpu_data->shared_cpu_map);
+ kfree(cpu_data);
+ policy->driver_data = NULL;
+}
+
+static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+ struct cppc_cpudata *cpu_data;
+ struct cppc_perf_caps *caps;
+ int ret;
+
+ cpu_data = cppc_cpufreq_get_cpu_data(cpu);
+ if (!cpu_data) {
+ pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu);
+ return -ENODEV;
+ }
+ caps = &cpu_data->perf_caps;
+ policy->driver_data = cpu_data;
+
+ /*
+ * Set min to lowest nonlinear perf to avoid any efficiency penalty (see
+ * Section 8.4.7.1.1.5 of ACPI 6.1 spec)
+ */
+ policy->min = cppc_cpufreq_perf_to_khz(cpu_data,
+ caps->lowest_nonlinear_perf);
+ policy->max = cppc_cpufreq_perf_to_khz(cpu_data,
+ caps->nominal_perf);
+
+ /*
+ * Set cpuinfo.min_freq to Lowest to make the full range of performance
+ * available if userspace wants to use any perf between lowest & lowest
+ * nonlinear perf
+ */
+ policy->cpuinfo.min_freq = cppc_cpufreq_perf_to_khz(cpu_data,
+ caps->lowest_perf);
+ policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu_data,
+ caps->nominal_perf);
+
+ policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu);
+ policy->shared_type = cpu_data->shared_type;
+
+ switch (policy->shared_type) {
+ case CPUFREQ_SHARED_TYPE_HW:
+ case CPUFREQ_SHARED_TYPE_NONE:
+ /* Nothing to be done - we'll have a policy for each CPU */
+ break;
+ case CPUFREQ_SHARED_TYPE_ANY:
+ /*
+ * All CPUs in the domain will share a policy and all cpufreq
+ * operations will use a single cppc_cpudata structure stored
+ * in policy->driver_data.
+ */
+ cpumask_copy(policy->cpus, cpu_data->shared_cpu_map);
+ break;
+ default:
+ pr_debug("Unsupported CPU co-ord type: %d\n",
+ policy->shared_type);
+ ret = -EFAULT;
+ goto out;
+ }
+
+ policy->fast_switch_possible = cppc_allow_fast_switch();
+ policy->dvfs_possible_from_any_cpu = true;
+
+ /*
+ * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
+ * is supported.
+ */
+ if (caps->highest_perf > caps->nominal_perf)
+ boost_supported = true;
+
+ /* Set policy->cur to max now. The governors will adjust later. */
+ policy->cur = cppc_cpufreq_perf_to_khz(cpu_data, caps->highest_perf);
+ cpu_data->perf_ctrls.desired_perf = caps->highest_perf;
+
+ ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
+ if (ret) {
+ pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
+ caps->highest_perf, cpu, ret);
+ goto out;
+ }
+
+ cppc_cpufreq_cpu_fie_init(policy);
+ return 0;
+
+out:
+ cppc_cpufreq_put_cpu_data(policy);
+ return ret;
+}
+
+static int cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+ struct cppc_perf_caps *caps = &cpu_data->perf_caps;
+ unsigned int cpu = policy->cpu;
+ int ret;
+
+ cppc_cpufreq_cpu_fie_exit(policy);
+
+ cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;
+
+ ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
+ if (ret)
+ pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
+ caps->lowest_perf, cpu, ret);
+
+ cppc_cpufreq_put_cpu_data(policy);
+ return 0;
+}
+
+static inline u64 get_delta(u64 t1, u64 t0)
+{
+ if (t1 > t0 || t0 > ~(u32)0)
+ return t1 - t0;
+
+ return (u32)t1 - (u32)t0;
+}
+
+static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
+ struct cppc_perf_fb_ctrs *fb_ctrs_t0,
+ struct cppc_perf_fb_ctrs *fb_ctrs_t1)
+{
+ u64 delta_reference, delta_delivered;
+ u64 reference_perf;
+
+ reference_perf = fb_ctrs_t0->reference_perf;
+
+ delta_reference = get_delta(fb_ctrs_t1->reference,
+ fb_ctrs_t0->reference);
+ delta_delivered = get_delta(fb_ctrs_t1->delivered,
+ fb_ctrs_t0->delivered);
+
+ /* Check to avoid divide-by zero and invalid delivered_perf */
+ if (!delta_reference || !delta_delivered)
+ return cpu_data->perf_ctrls.desired_perf;
+
+ return (reference_perf * delta_delivered) / delta_reference;
+}
+
+static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
+{
+ struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+ u64 delivered_perf;
+ int ret;
+
+ cpufreq_cpu_put(policy);
+
+ ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t0);
+ if (ret)
+ return 0;
+
+ udelay(2); /* 2usec delay between sampling */
+
+ ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t1);
+ if (ret)
+ return 0;
+
+ delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0,
+ &fb_ctrs_t1);
+
+ return cppc_cpufreq_perf_to_khz(cpu_data, delivered_perf);
+}
+
+static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+ struct cppc_perf_caps *caps = &cpu_data->perf_caps;
+ int ret;
+
+ if (!boost_supported) {
+ pr_err("BOOST not supported by CPU or firmware\n");
+ return -EINVAL;
+ }
+
+ if (state)
+ policy->max = cppc_cpufreq_perf_to_khz(cpu_data,
+ caps->highest_perf);
+ else
+ policy->max = cppc_cpufreq_perf_to_khz(cpu_data,
+ caps->nominal_perf);
+ policy->cpuinfo.max_freq = policy->max;
+
+ ret = freq_qos_update_request(policy->max_freq_req, policy->max);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
+{
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+
+ return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf);
+}
+cpufreq_freq_attr_ro(freqdomain_cpus);
+
+static struct freq_attr *cppc_cpufreq_attr[] = {
+ &freqdomain_cpus,
+ NULL,
+};
+
+static struct cpufreq_driver cppc_cpufreq_driver = {
+ .flags = CPUFREQ_CONST_LOOPS,
+ .verify = cppc_verify_policy,
+ .target = cppc_cpufreq_set_target,
+ .get = cppc_cpufreq_get_rate,
+ .fast_switch = cppc_cpufreq_fast_switch,
+ .init = cppc_cpufreq_cpu_init,
+ .exit = cppc_cpufreq_cpu_exit,
+ .set_boost = cppc_cpufreq_set_boost,
+ .attr = cppc_cpufreq_attr,
+ .name = "cppc_cpufreq",
+};
+
+/*
+ * HISI platform does not support delivered performance counter and
+ * reference performance counter. It can calculate the performance using the
+ * platform specific mechanism. We reuse the desired performance register to
+ * store the real performance calculated by the platform.
+ */
+static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+ struct cppc_cpudata *cpu_data = policy->driver_data;
+ u64 desired_perf;
+ int ret;
+
+ cpufreq_cpu_put(policy);
+
+ ret = cppc_get_desired_perf(cpu, &desired_perf);
+ if (ret < 0)
+ return -EIO;
+
+ return cppc_cpufreq_perf_to_khz(cpu_data, desired_perf);
+}
+
+static void cppc_check_hisi_workaround(void)
+{
+ struct acpi_table_header *tbl;
+ acpi_status status = AE_OK;
+ int i;
+
+ status = acpi_get_table(ACPI_SIG_PCCT, 0, &tbl);
+ if (ACPI_FAILURE(status) || !tbl)
+ return;
+
+ for (i = 0; i < ARRAY_SIZE(wa_info); i++) {
+ if (!memcmp(wa_info[i].oem_id, tbl->oem_id, ACPI_OEM_ID_SIZE) &&
+ !memcmp(wa_info[i].oem_table_id, tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
+ wa_info[i].oem_revision == tbl->oem_revision) {
+ /* Overwrite the get() callback */
+ cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate;
+ fie_disabled = FIE_DISABLED;
+ break;
+ }
+ }
+
+ acpi_put_table(tbl);
+}
+
+static int __init cppc_cpufreq_init(void)
+{
+ int ret;
+
+ if (!acpi_cpc_valid())
+ return -ENODEV;
+
+ cppc_check_hisi_workaround();
+ cppc_freq_invariance_init();
+ populate_efficiency_class();
+
+ ret = cpufreq_register_driver(&cppc_cpufreq_driver);
+ if (ret)
+ cppc_freq_invariance_exit();
+
+ return ret;
+}
+
+static inline void free_cpu_data(void)
+{
+ struct cppc_cpudata *iter, *tmp;
+
+ list_for_each_entry_safe(iter, tmp, &cpu_data_list, node) {
+ free_cpumask_var(iter->shared_cpu_map);
+ list_del(&iter->node);
+ kfree(iter);
+ }
+
+}
+
+static void __exit cppc_cpufreq_exit(void)
+{
+ cpufreq_unregister_driver(&cppc_cpufreq_driver);
+ cppc_freq_invariance_exit();
+
+ free_cpu_data();
+}
+
+module_exit(cppc_cpufreq_exit);
+MODULE_AUTHOR("Ashwin Chaugule");
+MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec");
+MODULE_LICENSE("GPL");
+
+late_initcall(cppc_cpufreq_init);
+
+static const struct acpi_device_id cppc_acpi_ids[] __used = {
+ {ACPI_PROCESSOR_DEVICE_HID, },
+ {}
+};
+
+MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids);
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
new file mode 100644
index 0000000000..fb2875ce1f
--- /dev/null
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2016 Linaro.
+ * Viresh Kumar <viresh.kumar@linaro.org>
+ */
+
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "cpufreq-dt.h"
+
+/*
+ * Machines for which the cpufreq device is *always* created, mostly used for
+ * platforms using "operating-points" (V1) property.
+ */
+static const struct of_device_id allowlist[] __initconst = {
+ { .compatible = "allwinner,sun4i-a10", },
+ { .compatible = "allwinner,sun5i-a10s", },
+ { .compatible = "allwinner,sun5i-a13", },
+ { .compatible = "allwinner,sun5i-r8", },
+ { .compatible = "allwinner,sun6i-a31", },
+ { .compatible = "allwinner,sun6i-a31s", },
+ { .compatible = "allwinner,sun7i-a20", },
+ { .compatible = "allwinner,sun8i-a23", },
+ { .compatible = "allwinner,sun8i-a83t", },
+ { .compatible = "allwinner,sun8i-h3", },
+
+ { .compatible = "apm,xgene-shadowcat", },
+
+ { .compatible = "arm,integrator-ap", },
+ { .compatible = "arm,integrator-cp", },
+
+ { .compatible = "hisilicon,hi3660", },
+
+ { .compatible = "fsl,imx27", },
+ { .compatible = "fsl,imx51", },
+ { .compatible = "fsl,imx53", },
+
+ { .compatible = "marvell,berlin", },
+ { .compatible = "marvell,pxa250", },
+ { .compatible = "marvell,pxa270", },
+
+ { .compatible = "samsung,exynos3250", },
+ { .compatible = "samsung,exynos4210", },
+ { .compatible = "samsung,exynos5250", },
+#ifndef CONFIG_BL_SWITCHER
+ { .compatible = "samsung,exynos5800", },
+#endif
+
+ { .compatible = "renesas,emev2", },
+ { .compatible = "renesas,r7s72100", },
+ { .compatible = "renesas,r8a73a4", },
+ { .compatible = "renesas,r8a7740", },
+ { .compatible = "renesas,r8a7742", },
+ { .compatible = "renesas,r8a7743", },
+ { .compatible = "renesas,r8a7744", },
+ { .compatible = "renesas,r8a7745", },
+ { .compatible = "renesas,r8a7778", },
+ { .compatible = "renesas,r8a7779", },
+ { .compatible = "renesas,r8a7790", },
+ { .compatible = "renesas,r8a7791", },
+ { .compatible = "renesas,r8a7792", },
+ { .compatible = "renesas,r8a7793", },
+ { .compatible = "renesas,r8a7794", },
+ { .compatible = "renesas,sh73a0", },
+
+ { .compatible = "rockchip,rk2928", },
+ { .compatible = "rockchip,rk3036", },
+ { .compatible = "rockchip,rk3066a", },
+ { .compatible = "rockchip,rk3066b", },
+ { .compatible = "rockchip,rk3188", },
+ { .compatible = "rockchip,rk3228", },
+ { .compatible = "rockchip,rk3288", },
+ { .compatible = "rockchip,rk3328", },
+ { .compatible = "rockchip,rk3366", },
+ { .compatible = "rockchip,rk3368", },
+ { .compatible = "rockchip,rk3399",
+ .data = &(struct cpufreq_dt_platform_data)
+ { .have_governor_per_policy = true, },
+ },
+
+ { .compatible = "st-ericsson,u8500", },
+ { .compatible = "st-ericsson,u8540", },
+ { .compatible = "st-ericsson,u9500", },
+ { .compatible = "st-ericsson,u9540", },
+
+ { .compatible = "starfive,jh7110", },
+
+ { .compatible = "ti,omap2", },
+ { .compatible = "ti,omap4", },
+ { .compatible = "ti,omap5", },
+
+ { .compatible = "xlnx,zynq-7000", },
+ { .compatible = "xlnx,zynqmp", },
+
+ { }
+};
+
+/*
+ * Machines for which the cpufreq device is *not* created, mostly used for
+ * platforms using "operating-points-v2" property.
+ */
+static const struct of_device_id blocklist[] __initconst = {
+ { .compatible = "allwinner,sun50i-h6", },
+
+ { .compatible = "apple,arm-platform", },
+
+ { .compatible = "arm,vexpress", },
+
+ { .compatible = "calxeda,highbank", },
+ { .compatible = "calxeda,ecx-2000", },
+
+ { .compatible = "fsl,imx7ulp", },
+ { .compatible = "fsl,imx7d", },
+ { .compatible = "fsl,imx7s", },
+ { .compatible = "fsl,imx8mq", },
+ { .compatible = "fsl,imx8mm", },
+ { .compatible = "fsl,imx8mn", },
+ { .compatible = "fsl,imx8mp", },
+
+ { .compatible = "marvell,armadaxp", },
+
+ { .compatible = "mediatek,mt2701", },
+ { .compatible = "mediatek,mt2712", },
+ { .compatible = "mediatek,mt7622", },
+ { .compatible = "mediatek,mt7623", },
+ { .compatible = "mediatek,mt8167", },
+ { .compatible = "mediatek,mt817x", },
+ { .compatible = "mediatek,mt8173", },
+ { .compatible = "mediatek,mt8176", },
+ { .compatible = "mediatek,mt8183", },
+ { .compatible = "mediatek,mt8186", },
+ { .compatible = "mediatek,mt8365", },
+ { .compatible = "mediatek,mt8516", },
+
+ { .compatible = "nvidia,tegra20", },
+ { .compatible = "nvidia,tegra30", },
+ { .compatible = "nvidia,tegra124", },
+ { .compatible = "nvidia,tegra210", },
+ { .compatible = "nvidia,tegra234", },
+
+ { .compatible = "qcom,apq8096", },
+ { .compatible = "qcom,msm8996", },
+ { .compatible = "qcom,msm8998", },
+ { .compatible = "qcom,qcm2290", },
+ { .compatible = "qcom,qcs404", },
+ { .compatible = "qcom,qdu1000", },
+ { .compatible = "qcom,sa8155p" },
+ { .compatible = "qcom,sa8540p" },
+ { .compatible = "qcom,sa8775p" },
+ { .compatible = "qcom,sc7180", },
+ { .compatible = "qcom,sc7280", },
+ { .compatible = "qcom,sc8180x", },
+ { .compatible = "qcom,sc8280xp", },
+ { .compatible = "qcom,sdm845", },
+ { .compatible = "qcom,sdx75", },
+ { .compatible = "qcom,sm6115", },
+ { .compatible = "qcom,sm6350", },
+ { .compatible = "qcom,sm6375", },
+ { .compatible = "qcom,sm7225", },
+ { .compatible = "qcom,sm8150", },
+ { .compatible = "qcom,sm8250", },
+ { .compatible = "qcom,sm8350", },
+ { .compatible = "qcom,sm8450", },
+ { .compatible = "qcom,sm8550", },
+
+ { .compatible = "st,stih407", },
+ { .compatible = "st,stih410", },
+ { .compatible = "st,stih418", },
+
+ { .compatible = "ti,am33xx", },
+ { .compatible = "ti,am43", },
+ { .compatible = "ti,dra7", },
+ { .compatible = "ti,omap3", },
+ { .compatible = "ti,am625", },
+ { .compatible = "ti,am62a7", },
+
+ { .compatible = "qcom,ipq8064", },
+ { .compatible = "qcom,apq8064", },
+ { .compatible = "qcom,msm8974", },
+ { .compatible = "qcom,msm8960", },
+
+ { }
+};
+
+static bool __init cpu0_node_has_opp_v2_prop(void)
+{
+ struct device_node *np = of_cpu_device_node_get(0);
+ bool ret = false;
+
+ if (of_property_present(np, "operating-points-v2"))
+ ret = true;
+
+ of_node_put(np);
+ return ret;
+}
+
+static int __init cpufreq_dt_platdev_init(void)
+{
+ struct device_node *np = of_find_node_by_path("/");
+ const struct of_device_id *match;
+ const void *data = NULL;
+
+ if (!np)
+ return -ENODEV;
+
+ match = of_match_node(allowlist, np);
+ if (match) {
+ data = match->data;
+ goto create_pdev;
+ }
+
+ if (cpu0_node_has_opp_v2_prop() && !of_match_node(blocklist, np))
+ goto create_pdev;
+
+ of_node_put(np);
+ return -ENODEV;
+
+create_pdev:
+ of_node_put(np);
+ return PTR_ERR_OR_ZERO(platform_device_register_data(NULL, "cpufreq-dt",
+ -1, data,
+ sizeof(struct cpufreq_dt_platform_data)));
+}
+core_initcall(cpufreq_dt_platdev_init);
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
new file mode 100644
index 0000000000..8bd6e5e8f1
--- /dev/null
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Freescale Semiconductor, Inc.
+ *
+ * Copyright (C) 2014 Linaro.
+ * Viresh Kumar <viresh.kumar@linaro.org>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pm_opp.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+#include <linux/thermal.h>
+
+#include "cpufreq-dt.h"
+
+struct private_data {
+ struct list_head node;
+
+ cpumask_var_t cpus;
+ struct device *cpu_dev;
+ struct cpufreq_frequency_table *freq_table;
+ bool have_static_opps;
+ int opp_token;
+};
+
+static LIST_HEAD(priv_list);
+
+static struct freq_attr *cpufreq_dt_attr[] = {
+ &cpufreq_freq_attr_scaling_available_freqs,
+ NULL, /* Extra space for boost-attr if required */
+ NULL,
+};
+
+static struct private_data *cpufreq_dt_find_data(int cpu)
+{
+ struct private_data *priv;
+
+ list_for_each_entry(priv, &priv_list, node) {
+ if (cpumask_test_cpu(cpu, priv->cpus))
+ return priv;
+ }
+
+ return NULL;
+}
+
+static int set_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ struct private_data *priv = policy->driver_data;
+ unsigned long freq = policy->freq_table[index].frequency;
+
+ return dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000);
+}
+
+/*
+ * An earlier version of opp-v1 bindings used to name the regulator
+ * "cpu0-supply", we still need to handle that for backwards compatibility.
+ */
+static const char *find_supply_name(struct device *dev)
+{
+ struct device_node *np;
+ struct property *pp;
+ int cpu = dev->id;
+ const char *name = NULL;
+
+ np = of_node_get(dev->of_node);
+
+ /* This must be valid for sure */
+ if (WARN_ON(!np))
+ return NULL;
+
+ /* Try "cpu0" for older DTs */
+ if (!cpu) {
+ pp = of_find_property(np, "cpu0-supply", NULL);
+ if (pp) {
+ name = "cpu0";
+ goto node_put;
+ }
+ }
+
+ pp = of_find_property(np, "cpu-supply", NULL);
+ if (pp) {
+ name = "cpu";
+ goto node_put;
+ }
+
+ dev_dbg(dev, "no regulator for cpu%d\n", cpu);
+node_put:
+ of_node_put(np);
+ return name;
+}
+
+static int cpufreq_init(struct cpufreq_policy *policy)
+{
+ struct private_data *priv;
+ struct device *cpu_dev;
+ struct clk *cpu_clk;
+ unsigned int transition_latency;
+ int ret;
+
+ priv = cpufreq_dt_find_data(policy->cpu);
+ if (!priv) {
+ pr_err("failed to find data for cpu%d\n", policy->cpu);
+ return -ENODEV;
+ }
+ cpu_dev = priv->cpu_dev;
+
+ cpu_clk = clk_get(cpu_dev, NULL);
+ if (IS_ERR(cpu_clk)) {
+ ret = PTR_ERR(cpu_clk);
+ dev_err(cpu_dev, "%s: failed to get clk: %d\n", __func__, ret);
+ return ret;
+ }
+
+ transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
+ if (!transition_latency)
+ transition_latency = CPUFREQ_ETERNAL;
+
+ cpumask_copy(policy->cpus, priv->cpus);
+ policy->driver_data = priv;
+ policy->clk = cpu_clk;
+ policy->freq_table = priv->freq_table;
+ policy->suspend_freq = dev_pm_opp_get_suspend_opp_freq(cpu_dev) / 1000;
+ policy->cpuinfo.transition_latency = transition_latency;
+ policy->dvfs_possible_from_any_cpu = true;
+
+ /* Support turbo/boost mode */
+ if (policy_has_boost_freq(policy)) {
+ /* This gets disabled by core on driver unregister */
+ ret = cpufreq_enable_boost_support();
+ if (ret)
+ goto out_clk_put;
+ cpufreq_dt_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
+ }
+
+ return 0;
+
+out_clk_put:
+ clk_put(cpu_clk);
+
+ return ret;
+}
+
+static int cpufreq_online(struct cpufreq_policy *policy)
+{
+ /* We did light-weight tear down earlier, nothing to do here */
+ return 0;
+}
+
+static int cpufreq_offline(struct cpufreq_policy *policy)
+{
+ /*
+ * Preserve policy->driver_data and don't free resources on light-weight
+ * tear down.
+ */
+ return 0;
+}
+
+static int cpufreq_exit(struct cpufreq_policy *policy)
+{
+ clk_put(policy->clk);
+ return 0;
+}
+
+static struct cpufreq_driver dt_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = set_target,
+ .get = cpufreq_generic_get,
+ .init = cpufreq_init,
+ .exit = cpufreq_exit,
+ .online = cpufreq_online,
+ .offline = cpufreq_offline,
+ .register_em = cpufreq_register_em_with_opp,
+ .name = "cpufreq-dt",
+ .attr = cpufreq_dt_attr,
+ .suspend = cpufreq_generic_suspend,
+};
+
+static int dt_cpufreq_early_init(struct device *dev, int cpu)
+{
+ struct private_data *priv;
+ struct device *cpu_dev;
+ bool fallback = false;
+ const char *reg_name[] = { NULL, NULL };
+ int ret;
+
+ /* Check if this CPU is already covered by some other policy */
+ if (cpufreq_dt_find_data(cpu))
+ return 0;
+
+ cpu_dev = get_cpu_device(cpu);
+ if (!cpu_dev)
+ return -EPROBE_DEFER;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ if (!alloc_cpumask_var(&priv->cpus, GFP_KERNEL))
+ return -ENOMEM;
+
+ cpumask_set_cpu(cpu, priv->cpus);
+ priv->cpu_dev = cpu_dev;
+
+ /*
+ * OPP layer will be taking care of regulators now, but it needs to know
+ * the name of the regulator first.
+ */
+ reg_name[0] = find_supply_name(cpu_dev);
+ if (reg_name[0]) {
+ priv->opp_token = dev_pm_opp_set_regulators(cpu_dev, reg_name);
+ if (priv->opp_token < 0) {
+ ret = dev_err_probe(cpu_dev, priv->opp_token,
+ "failed to set regulators\n");
+ goto free_cpumask;
+ }
+ }
+
+ /* Get OPP-sharing information from "operating-points-v2" bindings */
+ ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->cpus);
+ if (ret) {
+ if (ret != -ENOENT)
+ goto out;
+
+ /*
+ * operating-points-v2 not supported, fallback to all CPUs share
+ * OPP for backward compatibility if the platform hasn't set
+ * sharing CPUs.
+ */
+ if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus))
+ fallback = true;
+ }
+
+ /*
+ * Initialize OPP tables for all priv->cpus. They will be shared by
+ * all CPUs which have marked their CPUs shared with OPP bindings.
+ *
+ * For platforms not using operating-points-v2 bindings, we do this
+ * before updating priv->cpus. Otherwise, we will end up creating
+ * duplicate OPPs for the CPUs.
+ *
+ * OPPs might be populated at runtime, don't fail for error here unless
+ * it is -EPROBE_DEFER.
+ */
+ ret = dev_pm_opp_of_cpumask_add_table(priv->cpus);
+ if (!ret) {
+ priv->have_static_opps = true;
+ } else if (ret == -EPROBE_DEFER) {
+ goto out;
+ }
+
+ /*
+ * The OPP table must be initialized, statically or dynamically, by this
+ * point.
+ */
+ ret = dev_pm_opp_get_opp_count(cpu_dev);
+ if (ret <= 0) {
+ dev_err(cpu_dev, "OPP table can't be empty\n");
+ ret = -ENODEV;
+ goto out;
+ }
+
+ if (fallback) {
+ cpumask_setall(priv->cpus);
+ ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus);
+ if (ret)
+ dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
+ __func__, ret);
+ }
+
+ ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &priv->freq_table);
+ if (ret) {
+ dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
+ goto out;
+ }
+
+ list_add(&priv->node, &priv_list);
+ return 0;
+
+out:
+ if (priv->have_static_opps)
+ dev_pm_opp_of_cpumask_remove_table(priv->cpus);
+ dev_pm_opp_put_regulators(priv->opp_token);
+free_cpumask:
+ free_cpumask_var(priv->cpus);
+ return ret;
+}
+
+static void dt_cpufreq_release(void)
+{
+ struct private_data *priv, *tmp;
+
+ list_for_each_entry_safe(priv, tmp, &priv_list, node) {
+ dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &priv->freq_table);
+ if (priv->have_static_opps)
+ dev_pm_opp_of_cpumask_remove_table(priv->cpus);
+ dev_pm_opp_put_regulators(priv->opp_token);
+ free_cpumask_var(priv->cpus);
+ list_del(&priv->node);
+ }
+}
+
+static int dt_cpufreq_probe(struct platform_device *pdev)
+{
+ struct cpufreq_dt_platform_data *data = dev_get_platdata(&pdev->dev);
+ int ret, cpu;
+
+ /* Request resources early so we can return in case of -EPROBE_DEFER */
+ for_each_possible_cpu(cpu) {
+ ret = dt_cpufreq_early_init(&pdev->dev, cpu);
+ if (ret)
+ goto err;
+ }
+
+ if (data) {
+ if (data->have_governor_per_policy)
+ dt_cpufreq_driver.flags |= CPUFREQ_HAVE_GOVERNOR_PER_POLICY;
+
+ dt_cpufreq_driver.resume = data->resume;
+ if (data->suspend)
+ dt_cpufreq_driver.suspend = data->suspend;
+ if (data->get_intermediate) {
+ dt_cpufreq_driver.target_intermediate = data->target_intermediate;
+ dt_cpufreq_driver.get_intermediate = data->get_intermediate;
+ }
+ }
+
+ ret = cpufreq_register_driver(&dt_cpufreq_driver);
+ if (ret) {
+ dev_err(&pdev->dev, "failed register driver: %d\n", ret);
+ goto err;
+ }
+
+ return 0;
+err:
+ dt_cpufreq_release();
+ return ret;
+}
+
+static void dt_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&dt_cpufreq_driver);
+ dt_cpufreq_release();
+}
+
+static struct platform_driver dt_cpufreq_platdrv = {
+ .driver = {
+ .name = "cpufreq-dt",
+ },
+ .probe = dt_cpufreq_probe,
+ .remove_new = dt_cpufreq_remove,
+};
+module_platform_driver(dt_cpufreq_platdrv);
+
+MODULE_ALIAS("platform:cpufreq-dt");
+MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>");
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_DESCRIPTION("Generic cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/cpufreq-dt.h b/drivers/cpufreq/cpufreq-dt.h
new file mode 100644
index 0000000000..28c8af7ec5
--- /dev/null
+++ b/drivers/cpufreq/cpufreq-dt.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2016 Linaro
+ * Viresh Kumar <viresh.kumar@linaro.org>
+ */
+
+#ifndef __CPUFREQ_DT_H__
+#define __CPUFREQ_DT_H__
+
+#include <linux/types.h>
+
+struct cpufreq_policy;
+
+struct cpufreq_dt_platform_data {
+ bool have_governor_per_policy;
+
+ unsigned int (*get_intermediate)(struct cpufreq_policy *policy,
+ unsigned int index);
+ int (*target_intermediate)(struct cpufreq_policy *policy,
+ unsigned int index);
+ int (*suspend)(struct cpufreq_policy *policy);
+ int (*resume)(struct cpufreq_policy *policy);
+};
+
+#endif /* __CPUFREQ_DT_H__ */
diff --git a/drivers/cpufreq/cpufreq-nforce2.c b/drivers/cpufreq/cpufreq-nforce2.c
new file mode 100644
index 0000000000..f7a7bcf6f5
--- /dev/null
+++ b/drivers/cpufreq/cpufreq-nforce2.c
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * (C) 2004-2006 Sebastian Witt <se.witt@gmx.net>
+ *
+ * Based upon reverse engineered information
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+
+#define NFORCE2_XTAL 25
+#define NFORCE2_BOOTFSB 0x48
+#define NFORCE2_PLLENABLE 0xa8
+#define NFORCE2_PLLREG 0xa4
+#define NFORCE2_PLLADR 0xa0
+#define NFORCE2_PLL(mul, div) (0x100000 | (mul << 8) | div)
+
+#define NFORCE2_MIN_FSB 50
+#define NFORCE2_SAFE_DISTANCE 50
+
+/* Delay in ms between FSB changes */
+/* #define NFORCE2_DELAY 10 */
+
+/*
+ * nforce2_chipset:
+ * FSB is changed using the chipset
+ */
+static struct pci_dev *nforce2_dev;
+
+/* fid:
+ * multiplier * 10
+ */
+static int fid;
+
+/* min_fsb, max_fsb:
+ * minimum and maximum FSB (= FSB at boot time)
+ */
+static int min_fsb;
+static int max_fsb;
+
+MODULE_AUTHOR("Sebastian Witt <se.witt@gmx.net>");
+MODULE_DESCRIPTION("nForce2 FSB changing cpufreq driver");
+MODULE_LICENSE("GPL");
+
+module_param(fid, int, 0444);
+module_param(min_fsb, int, 0444);
+
+MODULE_PARM_DESC(fid, "CPU multiplier to use (11.5 = 115)");
+MODULE_PARM_DESC(min_fsb,
+ "Minimum FSB to use, if not defined: current FSB - 50");
+
+/**
+ * nforce2_calc_fsb - calculate FSB
+ * @pll: PLL value
+ *
+ * Calculates FSB from PLL value
+ */
+static int nforce2_calc_fsb(int pll)
+{
+ unsigned char mul, div;
+
+ mul = (pll >> 8) & 0xff;
+ div = pll & 0xff;
+
+ if (div > 0)
+ return NFORCE2_XTAL * mul / div;
+
+ return 0;
+}
+
+/**
+ * nforce2_calc_pll - calculate PLL value
+ * @fsb: FSB
+ *
+ * Calculate PLL value for given FSB
+ */
+static int nforce2_calc_pll(unsigned int fsb)
+{
+ unsigned char xmul, xdiv;
+ unsigned char mul = 0, div = 0;
+ int tried = 0;
+
+ /* Try to calculate multiplier and divider up to 4 times */
+ while (((mul == 0) || (div == 0)) && (tried <= 3)) {
+ for (xdiv = 2; xdiv <= 0x80; xdiv++)
+ for (xmul = 1; xmul <= 0xfe; xmul++)
+ if (nforce2_calc_fsb(NFORCE2_PLL(xmul, xdiv)) ==
+ fsb + tried) {
+ mul = xmul;
+ div = xdiv;
+ }
+ tried++;
+ }
+
+ if ((mul == 0) || (div == 0))
+ return -1;
+
+ return NFORCE2_PLL(mul, div);
+}
+
+/**
+ * nforce2_write_pll - write PLL value to chipset
+ * @pll: PLL value
+ *
+ * Writes new FSB PLL value to chipset
+ */
+static void nforce2_write_pll(int pll)
+{
+ int temp;
+
+ /* Set the pll addr. to 0x00 */
+ pci_write_config_dword(nforce2_dev, NFORCE2_PLLADR, 0);
+
+ /* Now write the value in all 64 registers */
+ for (temp = 0; temp <= 0x3f; temp++)
+ pci_write_config_dword(nforce2_dev, NFORCE2_PLLREG, pll);
+}
+
+/**
+ * nforce2_fsb_read - Read FSB
+ *
+ * Read FSB from chipset
+ * If bootfsb != 0, return FSB at boot-time
+ */
+static unsigned int nforce2_fsb_read(int bootfsb)
+{
+ struct pci_dev *nforce2_sub5;
+ u32 fsb, temp = 0;
+
+ /* Get chipset boot FSB from subdevice 5 (FSB at boot-time) */
+ nforce2_sub5 = pci_get_subsys(PCI_VENDOR_ID_NVIDIA, 0x01EF,
+ PCI_ANY_ID, PCI_ANY_ID, NULL);
+ if (!nforce2_sub5)
+ return 0;
+
+ pci_read_config_dword(nforce2_sub5, NFORCE2_BOOTFSB, &fsb);
+ fsb /= 1000000;
+
+ /* Check if PLL register is already set */
+ pci_read_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8 *)&temp);
+
+ if (bootfsb || !temp)
+ return fsb;
+
+ /* Use PLL register FSB value */
+ pci_read_config_dword(nforce2_dev, NFORCE2_PLLREG, &temp);
+ fsb = nforce2_calc_fsb(temp);
+
+ return fsb;
+}
+
+/**
+ * nforce2_set_fsb - set new FSB
+ * @fsb: New FSB
+ *
+ * Sets new FSB
+ */
+static int nforce2_set_fsb(unsigned int fsb)
+{
+ u32 temp = 0;
+ unsigned int tfsb;
+ int diff;
+ int pll = 0;
+
+ if ((fsb > max_fsb) || (fsb < NFORCE2_MIN_FSB)) {
+ pr_err("FSB %d is out of range!\n", fsb);
+ return -EINVAL;
+ }
+
+ tfsb = nforce2_fsb_read(0);
+ if (!tfsb) {
+ pr_err("Error while reading the FSB\n");
+ return -EINVAL;
+ }
+
+ /* First write? Then set actual value */
+ pci_read_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8 *)&temp);
+ if (!temp) {
+ pll = nforce2_calc_pll(tfsb);
+
+ if (pll < 0)
+ return -EINVAL;
+
+ nforce2_write_pll(pll);
+ }
+
+ /* Enable write access */
+ temp = 0x01;
+ pci_write_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8)temp);
+
+ diff = tfsb - fsb;
+
+ if (!diff)
+ return 0;
+
+ while ((tfsb != fsb) && (tfsb <= max_fsb) && (tfsb >= min_fsb)) {
+ if (diff < 0)
+ tfsb++;
+ else
+ tfsb--;
+
+ /* Calculate the PLL reg. value */
+ pll = nforce2_calc_pll(tfsb);
+ if (pll == -1)
+ return -EINVAL;
+
+ nforce2_write_pll(pll);
+#ifdef NFORCE2_DELAY
+ mdelay(NFORCE2_DELAY);
+#endif
+ }
+
+ temp = 0x40;
+ pci_write_config_byte(nforce2_dev, NFORCE2_PLLADR, (u8)temp);
+
+ return 0;
+}
+
+/**
+ * nforce2_get - get the CPU frequency
+ * @cpu: CPU number
+ *
+ * Returns the CPU frequency
+ */
+static unsigned int nforce2_get(unsigned int cpu)
+{
+ if (cpu)
+ return 0;
+ return nforce2_fsb_read(0) * fid * 100;
+}
+
+/**
+ * nforce2_target - set a new CPUFreq policy
+ * @policy: new policy
+ * @target_freq: the target frequency
+ * @relation: how that frequency relates to achieved frequency
+ * (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H)
+ *
+ * Sets a new CPUFreq policy.
+ */
+static int nforce2_target(struct cpufreq_policy *policy,
+ unsigned int target_freq, unsigned int relation)
+{
+/* unsigned long flags; */
+ struct cpufreq_freqs freqs;
+ unsigned int target_fsb;
+
+ if ((target_freq > policy->max) || (target_freq < policy->min))
+ return -EINVAL;
+
+ target_fsb = target_freq / (fid * 100);
+
+ freqs.old = nforce2_get(policy->cpu);
+ freqs.new = target_fsb * fid * 100;
+
+ if (freqs.old == freqs.new)
+ return 0;
+
+ pr_debug("Old CPU frequency %d kHz, new %d kHz\n",
+ freqs.old, freqs.new);
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+
+ /* Disable IRQs */
+ /* local_irq_save(flags); */
+
+ if (nforce2_set_fsb(target_fsb) < 0)
+ pr_err("Changing FSB to %d failed\n", target_fsb);
+ else
+ pr_debug("Changed FSB successfully to %d\n",
+ target_fsb);
+
+ /* Enable IRQs */
+ /* local_irq_restore(flags); */
+
+ cpufreq_freq_transition_end(policy, &freqs, 0);
+
+ return 0;
+}
+
+/**
+ * nforce2_verify - verifies a new CPUFreq policy
+ * @policy: new policy
+ */
+static int nforce2_verify(struct cpufreq_policy_data *policy)
+{
+ unsigned int fsb_pol_max;
+
+ fsb_pol_max = policy->max / (fid * 100);
+
+ if (policy->min < (fsb_pol_max * fid * 100))
+ policy->max = (fsb_pol_max + 1) * fid * 100;
+
+ cpufreq_verify_within_cpu_limits(policy);
+ return 0;
+}
+
+static int nforce2_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int fsb;
+ unsigned int rfid;
+
+ /* capability check */
+ if (policy->cpu != 0)
+ return -ENODEV;
+
+ /* Get current FSB */
+ fsb = nforce2_fsb_read(0);
+
+ if (!fsb)
+ return -EIO;
+
+ /* FIX: Get FID from CPU */
+ if (!fid) {
+ if (!cpu_khz) {
+ pr_warn("cpu_khz not set, can't calculate multiplier!\n");
+ return -ENODEV;
+ }
+
+ fid = cpu_khz / (fsb * 100);
+ rfid = fid % 5;
+
+ if (rfid) {
+ if (rfid > 2)
+ fid += 5 - rfid;
+ else
+ fid -= rfid;
+ }
+ }
+
+ pr_info("FSB currently at %i MHz, FID %d.%d\n",
+ fsb, fid / 10, fid % 10);
+
+ /* Set maximum FSB to FSB at boot time */
+ max_fsb = nforce2_fsb_read(1);
+
+ if (!max_fsb)
+ return -EIO;
+
+ if (!min_fsb)
+ min_fsb = max_fsb - NFORCE2_SAFE_DISTANCE;
+
+ if (min_fsb < NFORCE2_MIN_FSB)
+ min_fsb = NFORCE2_MIN_FSB;
+
+ /* cpuinfo and default policy values */
+ policy->min = policy->cpuinfo.min_freq = min_fsb * fid * 100;
+ policy->max = policy->cpuinfo.max_freq = max_fsb * fid * 100;
+
+ return 0;
+}
+
+static int nforce2_cpu_exit(struct cpufreq_policy *policy)
+{
+ return 0;
+}
+
+static struct cpufreq_driver nforce2_driver = {
+ .name = "nforce2",
+ .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
+ .verify = nforce2_verify,
+ .target = nforce2_target,
+ .get = nforce2_get,
+ .init = nforce2_cpu_init,
+ .exit = nforce2_cpu_exit,
+};
+
+#ifdef MODULE
+static const struct pci_device_id nforce2_ids[] = {
+ { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2 },
+ {}
+};
+MODULE_DEVICE_TABLE(pci, nforce2_ids);
+#endif
+
+/**
+ * nforce2_detect_chipset - detect the Southbridge which contains FSB PLL logic
+ *
+ * Detects nForce2 A2 and C1 stepping
+ *
+ */
+static int nforce2_detect_chipset(void)
+{
+ nforce2_dev = pci_get_subsys(PCI_VENDOR_ID_NVIDIA,
+ PCI_DEVICE_ID_NVIDIA_NFORCE2,
+ PCI_ANY_ID, PCI_ANY_ID, NULL);
+
+ if (nforce2_dev == NULL)
+ return -ENODEV;
+
+ pr_info("Detected nForce2 chipset revision %X\n",
+ nforce2_dev->revision);
+ pr_info("FSB changing is maybe unstable and can lead to crashes and data loss\n");
+
+ return 0;
+}
+
+/**
+ * nforce2_init - initializes the nForce2 CPUFreq driver
+ *
+ * Initializes the nForce2 FSB support. Returns -ENODEV on unsupported
+ * devices, -EINVAL on problems during initialization, and zero on
+ * success.
+ */
+static int __init nforce2_init(void)
+{
+ /* TODO: do we need to detect the processor? */
+
+ /* detect chipset */
+ if (nforce2_detect_chipset()) {
+ pr_info("No nForce2 chipset\n");
+ return -ENODEV;
+ }
+
+ return cpufreq_register_driver(&nforce2_driver);
+}
+
+/**
+ * nforce2_exit - unregisters cpufreq module
+ *
+ * Unregisters nForce2 FSB change support.
+ */
+static void __exit nforce2_exit(void)
+{
+ cpufreq_unregister_driver(&nforce2_driver);
+}
+
+module_init(nforce2_init);
+module_exit(nforce2_exit);
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
new file mode 100644
index 0000000000..60ed89000e
--- /dev/null
+++ b/drivers/cpufreq/cpufreq.c
@@ -0,0 +1,3005 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/drivers/cpufreq/cpufreq.c
+ *
+ * Copyright (C) 2001 Russell King
+ * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
+ *
+ * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
+ * Added handling for CPU hotplug
+ * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
+ * Fix handling for CPU hotplug -- affected CPUs
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/cpu_cooling.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel_stat.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm_qos.h>
+#include <linux/slab.h>
+#include <linux/suspend.h>
+#include <linux/syscore_ops.h>
+#include <linux/tick.h>
+#include <linux/units.h>
+#include <trace/events/power.h>
+
+static LIST_HEAD(cpufreq_policy_list);
+
+/* Macros to iterate over CPU policies */
+#define for_each_suitable_policy(__policy, __active) \
+ list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
+ if ((__active) == !policy_is_inactive(__policy))
+
+#define for_each_active_policy(__policy) \
+ for_each_suitable_policy(__policy, true)
+#define for_each_inactive_policy(__policy) \
+ for_each_suitable_policy(__policy, false)
+
+/* Iterate over governors */
+static LIST_HEAD(cpufreq_governor_list);
+#define for_each_governor(__governor) \
+ list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
+
+static char default_governor[CPUFREQ_NAME_LEN];
+
+/*
+ * The "cpufreq driver" - the arch- or hardware-dependent low
+ * level driver of CPUFreq support, and its spinlock. This lock
+ * also protects the cpufreq_cpu_data array.
+ */
+static struct cpufreq_driver *cpufreq_driver;
+static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
+static DEFINE_RWLOCK(cpufreq_driver_lock);
+
+static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
+bool cpufreq_supports_freq_invariance(void)
+{
+ return static_branch_likely(&cpufreq_freq_invariance);
+}
+
+/* Flag to suspend/resume CPUFreq governors */
+static bool cpufreq_suspended;
+
+static inline bool has_target(void)
+{
+ return cpufreq_driver->target_index || cpufreq_driver->target;
+}
+
+bool has_target_index(void)
+{
+ return !!cpufreq_driver->target_index;
+}
+
+/* internal prototypes */
+static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
+static int cpufreq_init_governor(struct cpufreq_policy *policy);
+static void cpufreq_exit_governor(struct cpufreq_policy *policy);
+static void cpufreq_governor_limits(struct cpufreq_policy *policy);
+static int cpufreq_set_policy(struct cpufreq_policy *policy,
+ struct cpufreq_governor *new_gov,
+ unsigned int new_pol);
+static bool cpufreq_boost_supported(void);
+
+/*
+ * Two notifier lists: the "policy" list is involved in the
+ * validation process for a new CPU frequency policy; the
+ * "transition" list for kernel code that needs to handle
+ * changes to devices when the CPU clock speed changes.
+ * The mutex locks both lists.
+ */
+static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
+SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
+
+static int off __read_mostly;
+static int cpufreq_disabled(void)
+{
+ return off;
+}
+void disable_cpufreq(void)
+{
+ off = 1;
+}
+static DEFINE_MUTEX(cpufreq_governor_mutex);
+
+bool have_governor_per_policy(void)
+{
+ return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
+}
+EXPORT_SYMBOL_GPL(have_governor_per_policy);
+
+static struct kobject *cpufreq_global_kobject;
+
+struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
+{
+ if (have_governor_per_policy())
+ return &policy->kobj;
+ else
+ return cpufreq_global_kobject;
+}
+EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
+
+static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
+{
+ struct kernel_cpustat kcpustat;
+ u64 cur_wall_time;
+ u64 idle_time;
+ u64 busy_time;
+
+ cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
+
+ kcpustat_cpu_fetch(&kcpustat, cpu);
+
+ busy_time = kcpustat.cpustat[CPUTIME_USER];
+ busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
+ busy_time += kcpustat.cpustat[CPUTIME_IRQ];
+ busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
+ busy_time += kcpustat.cpustat[CPUTIME_STEAL];
+ busy_time += kcpustat.cpustat[CPUTIME_NICE];
+
+ idle_time = cur_wall_time - busy_time;
+ if (wall)
+ *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
+
+ return div_u64(idle_time, NSEC_PER_USEC);
+}
+
+u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
+{
+ u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
+
+ if (idle_time == -1ULL)
+ return get_cpu_idle_time_jiffy(cpu, wall);
+ else if (!io_busy)
+ idle_time += get_cpu_iowait_time_us(cpu, wall);
+
+ return idle_time;
+}
+EXPORT_SYMBOL_GPL(get_cpu_idle_time);
+
+/*
+ * This is a generic cpufreq init() routine which can be used by cpufreq
+ * drivers of SMP systems. It will do following:
+ * - validate & show freq table passed
+ * - set policies transition latency
+ * - policy->cpus with all possible CPUs
+ */
+void cpufreq_generic_init(struct cpufreq_policy *policy,
+ struct cpufreq_frequency_table *table,
+ unsigned int transition_latency)
+{
+ policy->freq_table = table;
+ policy->cpuinfo.transition_latency = transition_latency;
+
+ /*
+ * The driver only supports the SMP configuration where all processors
+ * share the clock and voltage and clock.
+ */
+ cpumask_setall(policy->cpus);
+}
+EXPORT_SYMBOL_GPL(cpufreq_generic_init);
+
+struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
+
+ return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
+}
+EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
+
+unsigned int cpufreq_generic_get(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+
+ if (!policy || IS_ERR(policy->clk)) {
+ pr_err("%s: No %s associated to cpu: %d\n",
+ __func__, policy ? "clk" : "policy", cpu);
+ return 0;
+ }
+
+ return clk_get_rate(policy->clk) / 1000;
+}
+EXPORT_SYMBOL_GPL(cpufreq_generic_get);
+
+/**
+ * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
+ * @cpu: CPU to find the policy for.
+ *
+ * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
+ * the kobject reference counter of that policy. Return a valid policy on
+ * success or NULL on failure.
+ *
+ * The policy returned by this function has to be released with the help of
+ * cpufreq_cpu_put() to balance its kobject reference counter properly.
+ */
+struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = NULL;
+ unsigned long flags;
+
+ if (WARN_ON(cpu >= nr_cpu_ids))
+ return NULL;
+
+ /* get the cpufreq driver */
+ read_lock_irqsave(&cpufreq_driver_lock, flags);
+
+ if (cpufreq_driver) {
+ /* get the CPU */
+ policy = cpufreq_cpu_get_raw(cpu);
+ if (policy)
+ kobject_get(&policy->kobj);
+ }
+
+ read_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ return policy;
+}
+EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
+
+/**
+ * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
+ * @policy: cpufreq policy returned by cpufreq_cpu_get().
+ */
+void cpufreq_cpu_put(struct cpufreq_policy *policy)
+{
+ kobject_put(&policy->kobj);
+}
+EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
+
+/**
+ * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
+ * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
+ */
+void cpufreq_cpu_release(struct cpufreq_policy *policy)
+{
+ if (WARN_ON(!policy))
+ return;
+
+ lockdep_assert_held(&policy->rwsem);
+
+ up_write(&policy->rwsem);
+
+ cpufreq_cpu_put(policy);
+}
+
+/**
+ * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
+ * @cpu: CPU to find the policy for.
+ *
+ * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
+ * if the policy returned by it is not NULL, acquire its rwsem for writing.
+ * Return the policy if it is active or release it and return NULL otherwise.
+ *
+ * The policy returned by this function has to be released with the help of
+ * cpufreq_cpu_release() in order to release its rwsem and balance its usage
+ * counter properly.
+ */
+struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+
+ if (!policy)
+ return NULL;
+
+ down_write(&policy->rwsem);
+
+ if (policy_is_inactive(policy)) {
+ cpufreq_cpu_release(policy);
+ return NULL;
+ }
+
+ return policy;
+}
+
+/*********************************************************************
+ * EXTERNALLY AFFECTING FREQUENCY CHANGES *
+ *********************************************************************/
+
+/**
+ * adjust_jiffies - Adjust the system "loops_per_jiffy".
+ * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
+ * @ci: Frequency change information.
+ *
+ * This function alters the system "loops_per_jiffy" for the clock
+ * speed change. Note that loops_per_jiffy cannot be updated on SMP
+ * systems as each CPU might be scaled differently. So, use the arch
+ * per-CPU loops_per_jiffy value wherever possible.
+ */
+static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
+{
+#ifndef CONFIG_SMP
+ static unsigned long l_p_j_ref;
+ static unsigned int l_p_j_ref_freq;
+
+ if (ci->flags & CPUFREQ_CONST_LOOPS)
+ return;
+
+ if (!l_p_j_ref_freq) {
+ l_p_j_ref = loops_per_jiffy;
+ l_p_j_ref_freq = ci->old;
+ pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
+ l_p_j_ref, l_p_j_ref_freq);
+ }
+ if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
+ loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
+ ci->new);
+ pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
+ loops_per_jiffy, ci->new);
+ }
+#endif
+}
+
+/**
+ * cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
+ * @policy: cpufreq policy to enable fast frequency switching for.
+ * @freqs: contain details of the frequency update.
+ * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
+ *
+ * This function calls the transition notifiers and adjust_jiffies().
+ *
+ * It is called twice on all CPU frequency changes that have external effects.
+ */
+static void cpufreq_notify_transition(struct cpufreq_policy *policy,
+ struct cpufreq_freqs *freqs,
+ unsigned int state)
+{
+ int cpu;
+
+ BUG_ON(irqs_disabled());
+
+ if (cpufreq_disabled())
+ return;
+
+ freqs->policy = policy;
+ freqs->flags = cpufreq_driver->flags;
+ pr_debug("notification %u of frequency transition to %u kHz\n",
+ state, freqs->new);
+
+ switch (state) {
+ case CPUFREQ_PRECHANGE:
+ /*
+ * Detect if the driver reported a value as "old frequency"
+ * which is not equal to what the cpufreq core thinks is
+ * "old frequency".
+ */
+ if (policy->cur && policy->cur != freqs->old) {
+ pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
+ freqs->old, policy->cur);
+ freqs->old = policy->cur;
+ }
+
+ srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
+ CPUFREQ_PRECHANGE, freqs);
+
+ adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
+ break;
+
+ case CPUFREQ_POSTCHANGE:
+ adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
+ pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
+ cpumask_pr_args(policy->cpus));
+
+ for_each_cpu(cpu, policy->cpus)
+ trace_cpu_frequency(freqs->new, cpu);
+
+ srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
+ CPUFREQ_POSTCHANGE, freqs);
+
+ cpufreq_stats_record_transition(policy, freqs->new);
+ policy->cur = freqs->new;
+ }
+}
+
+/* Do post notifications when there are chances that transition has failed */
+static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
+ struct cpufreq_freqs *freqs, int transition_failed)
+{
+ cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
+ if (!transition_failed)
+ return;
+
+ swap(freqs->old, freqs->new);
+ cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
+ cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
+}
+
+void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
+ struct cpufreq_freqs *freqs)
+{
+
+ /*
+ * Catch double invocations of _begin() which lead to self-deadlock.
+ * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
+ * doesn't invoke _begin() on their behalf, and hence the chances of
+ * double invocations are very low. Moreover, there are scenarios
+ * where these checks can emit false-positive warnings in these
+ * drivers; so we avoid that by skipping them altogether.
+ */
+ WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
+ && current == policy->transition_task);
+
+wait:
+ wait_event(policy->transition_wait, !policy->transition_ongoing);
+
+ spin_lock(&policy->transition_lock);
+
+ if (unlikely(policy->transition_ongoing)) {
+ spin_unlock(&policy->transition_lock);
+ goto wait;
+ }
+
+ policy->transition_ongoing = true;
+ policy->transition_task = current;
+
+ spin_unlock(&policy->transition_lock);
+
+ cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
+}
+EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
+
+void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
+ struct cpufreq_freqs *freqs, int transition_failed)
+{
+ if (WARN_ON(!policy->transition_ongoing))
+ return;
+
+ cpufreq_notify_post_transition(policy, freqs, transition_failed);
+
+ arch_set_freq_scale(policy->related_cpus,
+ policy->cur,
+ policy->cpuinfo.max_freq);
+
+ spin_lock(&policy->transition_lock);
+ policy->transition_ongoing = false;
+ policy->transition_task = NULL;
+ spin_unlock(&policy->transition_lock);
+
+ wake_up(&policy->transition_wait);
+}
+EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
+
+/*
+ * Fast frequency switching status count. Positive means "enabled", negative
+ * means "disabled" and 0 means "not decided yet".
+ */
+static int cpufreq_fast_switch_count;
+static DEFINE_MUTEX(cpufreq_fast_switch_lock);
+
+static void cpufreq_list_transition_notifiers(void)
+{
+ struct notifier_block *nb;
+
+ pr_info("Registered transition notifiers:\n");
+
+ mutex_lock(&cpufreq_transition_notifier_list.mutex);
+
+ for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
+ pr_info("%pS\n", nb->notifier_call);
+
+ mutex_unlock(&cpufreq_transition_notifier_list.mutex);
+}
+
+/**
+ * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
+ * @policy: cpufreq policy to enable fast frequency switching for.
+ *
+ * Try to enable fast frequency switching for @policy.
+ *
+ * The attempt will fail if there is at least one transition notifier registered
+ * at this point, as fast frequency switching is quite fundamentally at odds
+ * with transition notifiers. Thus if successful, it will make registration of
+ * transition notifiers fail going forward.
+ */
+void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
+{
+ lockdep_assert_held(&policy->rwsem);
+
+ if (!policy->fast_switch_possible)
+ return;
+
+ mutex_lock(&cpufreq_fast_switch_lock);
+ if (cpufreq_fast_switch_count >= 0) {
+ cpufreq_fast_switch_count++;
+ policy->fast_switch_enabled = true;
+ } else {
+ pr_warn("CPU%u: Fast frequency switching not enabled\n",
+ policy->cpu);
+ cpufreq_list_transition_notifiers();
+ }
+ mutex_unlock(&cpufreq_fast_switch_lock);
+}
+EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
+
+/**
+ * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
+ * @policy: cpufreq policy to disable fast frequency switching for.
+ */
+void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
+{
+ mutex_lock(&cpufreq_fast_switch_lock);
+ if (policy->fast_switch_enabled) {
+ policy->fast_switch_enabled = false;
+ if (!WARN_ON(cpufreq_fast_switch_count <= 0))
+ cpufreq_fast_switch_count--;
+ }
+ mutex_unlock(&cpufreq_fast_switch_lock);
+}
+EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
+
+static unsigned int __resolve_freq(struct cpufreq_policy *policy,
+ unsigned int target_freq, unsigned int relation)
+{
+ unsigned int idx;
+
+ target_freq = clamp_val(target_freq, policy->min, policy->max);
+
+ if (!policy->freq_table)
+ return target_freq;
+
+ idx = cpufreq_frequency_table_target(policy, target_freq, relation);
+ policy->cached_resolved_idx = idx;
+ policy->cached_target_freq = target_freq;
+ return policy->freq_table[idx].frequency;
+}
+
+/**
+ * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
+ * one.
+ * @policy: associated policy to interrogate
+ * @target_freq: target frequency to resolve.
+ *
+ * The target to driver frequency mapping is cached in the policy.
+ *
+ * Return: Lowest driver-supported frequency greater than or equal to the
+ * given target_freq, subject to policy (min/max) and driver limitations.
+ */
+unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ return __resolve_freq(policy, target_freq, CPUFREQ_RELATION_LE);
+}
+EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
+
+unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
+{
+ unsigned int latency;
+
+ if (policy->transition_delay_us)
+ return policy->transition_delay_us;
+
+ latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
+ if (latency) {
+ /*
+ * For platforms that can change the frequency very fast (< 10
+ * us), the above formula gives a decent transition delay. But
+ * for platforms where transition_latency is in milliseconds, it
+ * ends up giving unrealistic values.
+ *
+ * Cap the default transition delay to 10 ms, which seems to be
+ * a reasonable amount of time after which we should reevaluate
+ * the frequency.
+ */
+ return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
+ }
+
+ return LATENCY_MULTIPLIER;
+}
+EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
+
+/*********************************************************************
+ * SYSFS INTERFACE *
+ *********************************************************************/
+static ssize_t show_boost(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
+}
+
+static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret, enable;
+
+ ret = sscanf(buf, "%d", &enable);
+ if (ret != 1 || enable < 0 || enable > 1)
+ return -EINVAL;
+
+ if (cpufreq_boost_trigger_state(enable)) {
+ pr_err("%s: Cannot %s BOOST!\n",
+ __func__, enable ? "enable" : "disable");
+ return -EINVAL;
+ }
+
+ pr_debug("%s: cpufreq BOOST %s\n",
+ __func__, enable ? "enabled" : "disabled");
+
+ return count;
+}
+define_one_global_rw(boost);
+
+static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf)
+{
+ return sysfs_emit(buf, "%d\n", policy->boost_enabled);
+}
+
+static ssize_t store_local_boost(struct cpufreq_policy *policy,
+ const char *buf, size_t count)
+{
+ int ret, enable;
+
+ ret = kstrtoint(buf, 10, &enable);
+ if (ret || enable < 0 || enable > 1)
+ return -EINVAL;
+
+ if (!cpufreq_driver->boost_enabled)
+ return -EINVAL;
+
+ if (policy->boost_enabled == enable)
+ return count;
+
+ cpus_read_lock();
+ ret = cpufreq_driver->set_boost(policy, enable);
+ cpus_read_unlock();
+
+ if (ret)
+ return ret;
+
+ policy->boost_enabled = enable;
+
+ return count;
+}
+
+static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost);
+
+static struct cpufreq_governor *find_governor(const char *str_governor)
+{
+ struct cpufreq_governor *t;
+
+ for_each_governor(t)
+ if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
+ return t;
+
+ return NULL;
+}
+
+static struct cpufreq_governor *get_governor(const char *str_governor)
+{
+ struct cpufreq_governor *t;
+
+ mutex_lock(&cpufreq_governor_mutex);
+ t = find_governor(str_governor);
+ if (!t)
+ goto unlock;
+
+ if (!try_module_get(t->owner))
+ t = NULL;
+
+unlock:
+ mutex_unlock(&cpufreq_governor_mutex);
+
+ return t;
+}
+
+static unsigned int cpufreq_parse_policy(char *str_governor)
+{
+ if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN))
+ return CPUFREQ_POLICY_PERFORMANCE;
+
+ if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN))
+ return CPUFREQ_POLICY_POWERSAVE;
+
+ return CPUFREQ_POLICY_UNKNOWN;
+}
+
+/**
+ * cpufreq_parse_governor - parse a governor string only for has_target()
+ * @str_governor: Governor name.
+ */
+static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
+{
+ struct cpufreq_governor *t;
+
+ t = get_governor(str_governor);
+ if (t)
+ return t;
+
+ if (request_module("cpufreq_%s", str_governor))
+ return NULL;
+
+ return get_governor(str_governor);
+}
+
+/*
+ * cpufreq_per_cpu_attr_read() / show_##file_name() -
+ * print out cpufreq information
+ *
+ * Write out information from cpufreq_driver->policy[cpu]; object must be
+ * "unsigned int".
+ */
+
+#define show_one(file_name, object) \
+static ssize_t show_##file_name \
+(struct cpufreq_policy *policy, char *buf) \
+{ \
+ return sprintf(buf, "%u\n", policy->object); \
+}
+
+show_one(cpuinfo_min_freq, cpuinfo.min_freq);
+show_one(cpuinfo_max_freq, cpuinfo.max_freq);
+show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
+show_one(scaling_min_freq, min);
+show_one(scaling_max_freq, max);
+
+__weak unsigned int arch_freq_get_on_cpu(int cpu)
+{
+ return 0;
+}
+
+static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
+{
+ ssize_t ret;
+ unsigned int freq;
+
+ freq = arch_freq_get_on_cpu(policy->cpu);
+ if (freq)
+ ret = sprintf(buf, "%u\n", freq);
+ else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
+ ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
+ else
+ ret = sprintf(buf, "%u\n", policy->cur);
+ return ret;
+}
+
+/*
+ * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
+ */
+#define store_one(file_name, object) \
+static ssize_t store_##file_name \
+(struct cpufreq_policy *policy, const char *buf, size_t count) \
+{ \
+ unsigned long val; \
+ int ret; \
+ \
+ ret = kstrtoul(buf, 0, &val); \
+ if (ret) \
+ return ret; \
+ \
+ ret = freq_qos_update_request(policy->object##_freq_req, val);\
+ return ret >= 0 ? count : ret; \
+}
+
+store_one(scaling_min_freq, min);
+store_one(scaling_max_freq, max);
+
+/*
+ * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
+ */
+static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
+ char *buf)
+{
+ unsigned int cur_freq = __cpufreq_get(policy);
+
+ if (cur_freq)
+ return sprintf(buf, "%u\n", cur_freq);
+
+ return sprintf(buf, "<unknown>\n");
+}
+
+/*
+ * show_scaling_governor - show the current policy for the specified CPU
+ */
+static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
+{
+ if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
+ return sprintf(buf, "powersave\n");
+ else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
+ return sprintf(buf, "performance\n");
+ else if (policy->governor)
+ return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
+ policy->governor->name);
+ return -EINVAL;
+}
+
+/*
+ * store_scaling_governor - store policy for the specified CPU
+ */
+static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
+ const char *buf, size_t count)
+{
+ char str_governor[16];
+ int ret;
+
+ ret = sscanf(buf, "%15s", str_governor);
+ if (ret != 1)
+ return -EINVAL;
+
+ if (cpufreq_driver->setpolicy) {
+ unsigned int new_pol;
+
+ new_pol = cpufreq_parse_policy(str_governor);
+ if (!new_pol)
+ return -EINVAL;
+
+ ret = cpufreq_set_policy(policy, NULL, new_pol);
+ } else {
+ struct cpufreq_governor *new_gov;
+
+ new_gov = cpufreq_parse_governor(str_governor);
+ if (!new_gov)
+ return -EINVAL;
+
+ ret = cpufreq_set_policy(policy, new_gov,
+ CPUFREQ_POLICY_UNKNOWN);
+
+ module_put(new_gov->owner);
+ }
+
+ return ret ? ret : count;
+}
+
+/*
+ * show_scaling_driver - show the cpufreq driver currently loaded
+ */
+static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
+{
+ return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
+}
+
+/*
+ * show_scaling_available_governors - show the available CPUfreq governors
+ */
+static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
+ char *buf)
+{
+ ssize_t i = 0;
+ struct cpufreq_governor *t;
+
+ if (!has_target()) {
+ i += sprintf(buf, "performance powersave");
+ goto out;
+ }
+
+ mutex_lock(&cpufreq_governor_mutex);
+ for_each_governor(t) {
+ if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
+ - (CPUFREQ_NAME_LEN + 2)))
+ break;
+ i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
+ }
+ mutex_unlock(&cpufreq_governor_mutex);
+out:
+ i += sprintf(&buf[i], "\n");
+ return i;
+}
+
+ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
+{
+ ssize_t i = 0;
+ unsigned int cpu;
+
+ for_each_cpu(cpu, mask) {
+ i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u ", cpu);
+ if (i >= (PAGE_SIZE - 5))
+ break;
+ }
+
+ /* Remove the extra space at the end */
+ i--;
+
+ i += sprintf(&buf[i], "\n");
+ return i;
+}
+EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
+
+/*
+ * show_related_cpus - show the CPUs affected by each transition even if
+ * hw coordination is in use
+ */
+static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
+{
+ return cpufreq_show_cpus(policy->related_cpus, buf);
+}
+
+/*
+ * show_affected_cpus - show the CPUs affected by each transition
+ */
+static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
+{
+ return cpufreq_show_cpus(policy->cpus, buf);
+}
+
+static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
+ const char *buf, size_t count)
+{
+ unsigned int freq = 0;
+ unsigned int ret;
+
+ if (!policy->governor || !policy->governor->store_setspeed)
+ return -EINVAL;
+
+ ret = sscanf(buf, "%u", &freq);
+ if (ret != 1)
+ return -EINVAL;
+
+ policy->governor->store_setspeed(policy, freq);
+
+ return count;
+}
+
+static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
+{
+ if (!policy->governor || !policy->governor->show_setspeed)
+ return sprintf(buf, "<unsupported>\n");
+
+ return policy->governor->show_setspeed(policy, buf);
+}
+
+/*
+ * show_bios_limit - show the current cpufreq HW/BIOS limitation
+ */
+static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
+{
+ unsigned int limit;
+ int ret;
+ ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
+ if (!ret)
+ return sprintf(buf, "%u\n", limit);
+ return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
+}
+
+cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
+cpufreq_freq_attr_ro(cpuinfo_min_freq);
+cpufreq_freq_attr_ro(cpuinfo_max_freq);
+cpufreq_freq_attr_ro(cpuinfo_transition_latency);
+cpufreq_freq_attr_ro(scaling_available_governors);
+cpufreq_freq_attr_ro(scaling_driver);
+cpufreq_freq_attr_ro(scaling_cur_freq);
+cpufreq_freq_attr_ro(bios_limit);
+cpufreq_freq_attr_ro(related_cpus);
+cpufreq_freq_attr_ro(affected_cpus);
+cpufreq_freq_attr_rw(scaling_min_freq);
+cpufreq_freq_attr_rw(scaling_max_freq);
+cpufreq_freq_attr_rw(scaling_governor);
+cpufreq_freq_attr_rw(scaling_setspeed);
+
+static struct attribute *cpufreq_attrs[] = {
+ &cpuinfo_min_freq.attr,
+ &cpuinfo_max_freq.attr,
+ &cpuinfo_transition_latency.attr,
+ &scaling_min_freq.attr,
+ &scaling_max_freq.attr,
+ &affected_cpus.attr,
+ &related_cpus.attr,
+ &scaling_governor.attr,
+ &scaling_driver.attr,
+ &scaling_available_governors.attr,
+ &scaling_setspeed.attr,
+ NULL
+};
+ATTRIBUTE_GROUPS(cpufreq);
+
+#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
+#define to_attr(a) container_of(a, struct freq_attr, attr)
+
+static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+ struct cpufreq_policy *policy = to_policy(kobj);
+ struct freq_attr *fattr = to_attr(attr);
+ ssize_t ret = -EBUSY;
+
+ if (!fattr->show)
+ return -EIO;
+
+ down_read(&policy->rwsem);
+ if (likely(!policy_is_inactive(policy)))
+ ret = fattr->show(policy, buf);
+ up_read(&policy->rwsem);
+
+ return ret;
+}
+
+static ssize_t store(struct kobject *kobj, struct attribute *attr,
+ const char *buf, size_t count)
+{
+ struct cpufreq_policy *policy = to_policy(kobj);
+ struct freq_attr *fattr = to_attr(attr);
+ ssize_t ret = -EBUSY;
+
+ if (!fattr->store)
+ return -EIO;
+
+ down_write(&policy->rwsem);
+ if (likely(!policy_is_inactive(policy)))
+ ret = fattr->store(policy, buf, count);
+ up_write(&policy->rwsem);
+
+ return ret;
+}
+
+static void cpufreq_sysfs_release(struct kobject *kobj)
+{
+ struct cpufreq_policy *policy = to_policy(kobj);
+ pr_debug("last reference is dropped\n");
+ complete(&policy->kobj_unregister);
+}
+
+static const struct sysfs_ops sysfs_ops = {
+ .show = show,
+ .store = store,
+};
+
+static const struct kobj_type ktype_cpufreq = {
+ .sysfs_ops = &sysfs_ops,
+ .default_groups = cpufreq_groups,
+ .release = cpufreq_sysfs_release,
+};
+
+static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
+ struct device *dev)
+{
+ if (unlikely(!dev))
+ return;
+
+ if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
+ return;
+
+ dev_dbg(dev, "%s: Adding symlink\n", __func__);
+ if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
+ dev_err(dev, "cpufreq symlink creation failed\n");
+}
+
+static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu,
+ struct device *dev)
+{
+ dev_dbg(dev, "%s: Removing symlink\n", __func__);
+ sysfs_remove_link(&dev->kobj, "cpufreq");
+ cpumask_clear_cpu(cpu, policy->real_cpus);
+}
+
+static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
+{
+ struct freq_attr **drv_attr;
+ int ret = 0;
+
+ /* set up files for this cpu device */
+ drv_attr = cpufreq_driver->attr;
+ while (drv_attr && *drv_attr) {
+ ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
+ if (ret)
+ return ret;
+ drv_attr++;
+ }
+ if (cpufreq_driver->get) {
+ ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
+ if (ret)
+ return ret;
+ }
+
+ ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
+ if (ret)
+ return ret;
+
+ if (cpufreq_driver->bios_limit) {
+ ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
+ if (ret)
+ return ret;
+ }
+
+ if (cpufreq_boost_supported()) {
+ ret = sysfs_create_file(&policy->kobj, &local_boost.attr);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int cpufreq_init_policy(struct cpufreq_policy *policy)
+{
+ struct cpufreq_governor *gov = NULL;
+ unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
+ int ret;
+
+ if (has_target()) {
+ /* Update policy governor to the one used before hotplug. */
+ gov = get_governor(policy->last_governor);
+ if (gov) {
+ pr_debug("Restoring governor %s for cpu %d\n",
+ gov->name, policy->cpu);
+ } else {
+ gov = get_governor(default_governor);
+ }
+
+ if (!gov) {
+ gov = cpufreq_default_governor();
+ __module_get(gov->owner);
+ }
+
+ } else {
+
+ /* Use the default policy if there is no last_policy. */
+ if (policy->last_policy) {
+ pol = policy->last_policy;
+ } else {
+ pol = cpufreq_parse_policy(default_governor);
+ /*
+ * In case the default governor is neither "performance"
+ * nor "powersave", fall back to the initial policy
+ * value set by the driver.
+ */
+ if (pol == CPUFREQ_POLICY_UNKNOWN)
+ pol = policy->policy;
+ }
+ if (pol != CPUFREQ_POLICY_PERFORMANCE &&
+ pol != CPUFREQ_POLICY_POWERSAVE)
+ return -ENODATA;
+ }
+
+ ret = cpufreq_set_policy(policy, gov, pol);
+ if (gov)
+ module_put(gov->owner);
+
+ return ret;
+}
+
+static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
+{
+ int ret = 0;
+
+ /* Has this CPU been taken care of already? */
+ if (cpumask_test_cpu(cpu, policy->cpus))
+ return 0;
+
+ down_write(&policy->rwsem);
+ if (has_target())
+ cpufreq_stop_governor(policy);
+
+ cpumask_set_cpu(cpu, policy->cpus);
+
+ if (has_target()) {
+ ret = cpufreq_start_governor(policy);
+ if (ret)
+ pr_err("%s: Failed to start governor\n", __func__);
+ }
+ up_write(&policy->rwsem);
+ return ret;
+}
+
+void refresh_frequency_limits(struct cpufreq_policy *policy)
+{
+ if (!policy_is_inactive(policy)) {
+ pr_debug("updating policy for CPU %u\n", policy->cpu);
+
+ cpufreq_set_policy(policy, policy->governor, policy->policy);
+ }
+}
+EXPORT_SYMBOL(refresh_frequency_limits);
+
+static void handle_update(struct work_struct *work)
+{
+ struct cpufreq_policy *policy =
+ container_of(work, struct cpufreq_policy, update);
+
+ pr_debug("handle_update for cpu %u called\n", policy->cpu);
+ down_write(&policy->rwsem);
+ refresh_frequency_limits(policy);
+ up_write(&policy->rwsem);
+}
+
+static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
+ void *data)
+{
+ struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
+
+ schedule_work(&policy->update);
+ return 0;
+}
+
+static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
+ void *data)
+{
+ struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
+
+ schedule_work(&policy->update);
+ return 0;
+}
+
+static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
+{
+ struct kobject *kobj;
+ struct completion *cmp;
+
+ down_write(&policy->rwsem);
+ cpufreq_stats_free_table(policy);
+ kobj = &policy->kobj;
+ cmp = &policy->kobj_unregister;
+ up_write(&policy->rwsem);
+ kobject_put(kobj);
+
+ /*
+ * We need to make sure that the underlying kobj is
+ * actually not referenced anymore by anybody before we
+ * proceed with unloading.
+ */
+ pr_debug("waiting for dropping of refcount\n");
+ wait_for_completion(cmp);
+ pr_debug("wait complete\n");
+}
+
+static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
+{
+ struct cpufreq_policy *policy;
+ struct device *dev = get_cpu_device(cpu);
+ int ret;
+
+ if (!dev)
+ return NULL;
+
+ policy = kzalloc(sizeof(*policy), GFP_KERNEL);
+ if (!policy)
+ return NULL;
+
+ if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
+ goto err_free_policy;
+
+ if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
+ goto err_free_cpumask;
+
+ if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
+ goto err_free_rcpumask;
+
+ init_completion(&policy->kobj_unregister);
+ ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
+ cpufreq_global_kobject, "policy%u", cpu);
+ if (ret) {
+ dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
+ /*
+ * The entire policy object will be freed below, but the extra
+ * memory allocated for the kobject name needs to be freed by
+ * releasing the kobject.
+ */
+ kobject_put(&policy->kobj);
+ goto err_free_real_cpus;
+ }
+
+ freq_constraints_init(&policy->constraints);
+
+ policy->nb_min.notifier_call = cpufreq_notifier_min;
+ policy->nb_max.notifier_call = cpufreq_notifier_max;
+
+ ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
+ &policy->nb_min);
+ if (ret) {
+ dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n",
+ ret, cpu);
+ goto err_kobj_remove;
+ }
+
+ ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
+ &policy->nb_max);
+ if (ret) {
+ dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n",
+ ret, cpu);
+ goto err_min_qos_notifier;
+ }
+
+ INIT_LIST_HEAD(&policy->policy_list);
+ init_rwsem(&policy->rwsem);
+ spin_lock_init(&policy->transition_lock);
+ init_waitqueue_head(&policy->transition_wait);
+ INIT_WORK(&policy->update, handle_update);
+
+ policy->cpu = cpu;
+ return policy;
+
+err_min_qos_notifier:
+ freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
+ &policy->nb_min);
+err_kobj_remove:
+ cpufreq_policy_put_kobj(policy);
+err_free_real_cpus:
+ free_cpumask_var(policy->real_cpus);
+err_free_rcpumask:
+ free_cpumask_var(policy->related_cpus);
+err_free_cpumask:
+ free_cpumask_var(policy->cpus);
+err_free_policy:
+ kfree(policy);
+
+ return NULL;
+}
+
+static void cpufreq_policy_free(struct cpufreq_policy *policy)
+{
+ unsigned long flags;
+ int cpu;
+
+ /*
+ * The callers must ensure the policy is inactive by now, to avoid any
+ * races with show()/store() callbacks.
+ */
+ if (unlikely(!policy_is_inactive(policy)))
+ pr_warn("%s: Freeing active policy\n", __func__);
+
+ /* Remove policy from list */
+ write_lock_irqsave(&cpufreq_driver_lock, flags);
+ list_del(&policy->policy_list);
+
+ for_each_cpu(cpu, policy->related_cpus)
+ per_cpu(cpufreq_cpu_data, cpu) = NULL;
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
+ &policy->nb_max);
+ freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
+ &policy->nb_min);
+
+ /* Cancel any pending policy->update work before freeing the policy. */
+ cancel_work_sync(&policy->update);
+
+ if (policy->max_freq_req) {
+ /*
+ * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY
+ * notification, since CPUFREQ_CREATE_POLICY notification was
+ * sent after adding max_freq_req earlier.
+ */
+ blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
+ CPUFREQ_REMOVE_POLICY, policy);
+ freq_qos_remove_request(policy->max_freq_req);
+ }
+
+ freq_qos_remove_request(policy->min_freq_req);
+ kfree(policy->min_freq_req);
+
+ cpufreq_policy_put_kobj(policy);
+ free_cpumask_var(policy->real_cpus);
+ free_cpumask_var(policy->related_cpus);
+ free_cpumask_var(policy->cpus);
+ kfree(policy);
+}
+
+static int cpufreq_online(unsigned int cpu)
+{
+ struct cpufreq_policy *policy;
+ bool new_policy;
+ unsigned long flags;
+ unsigned int j;
+ int ret;
+
+ pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
+
+ /* Check if this CPU already has a policy to manage it */
+ policy = per_cpu(cpufreq_cpu_data, cpu);
+ if (policy) {
+ WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
+ if (!policy_is_inactive(policy))
+ return cpufreq_add_policy_cpu(policy, cpu);
+
+ /* This is the only online CPU for the policy. Start over. */
+ new_policy = false;
+ down_write(&policy->rwsem);
+ policy->cpu = cpu;
+ policy->governor = NULL;
+ } else {
+ new_policy = true;
+ policy = cpufreq_policy_alloc(cpu);
+ if (!policy)
+ return -ENOMEM;
+ down_write(&policy->rwsem);
+ }
+
+ if (!new_policy && cpufreq_driver->online) {
+ /* Recover policy->cpus using related_cpus */
+ cpumask_copy(policy->cpus, policy->related_cpus);
+
+ ret = cpufreq_driver->online(policy);
+ if (ret) {
+ pr_debug("%s: %d: initialization failed\n", __func__,
+ __LINE__);
+ goto out_exit_policy;
+ }
+ } else {
+ cpumask_copy(policy->cpus, cpumask_of(cpu));
+
+ /*
+ * Call driver. From then on the cpufreq must be able
+ * to accept all calls to ->verify and ->setpolicy for this CPU.
+ */
+ ret = cpufreq_driver->init(policy);
+ if (ret) {
+ pr_debug("%s: %d: initialization failed\n", __func__,
+ __LINE__);
+ goto out_free_policy;
+ }
+
+ /*
+ * The initialization has succeeded and the policy is online.
+ * If there is a problem with its frequency table, take it
+ * offline and drop it.
+ */
+ ret = cpufreq_table_validate_and_sort(policy);
+ if (ret)
+ goto out_offline_policy;
+
+ /* related_cpus should at least include policy->cpus. */
+ cpumask_copy(policy->related_cpus, policy->cpus);
+ }
+
+ /*
+ * affected cpus must always be the one, which are online. We aren't
+ * managing offline cpus here.
+ */
+ cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
+
+ if (new_policy) {
+ for_each_cpu(j, policy->related_cpus) {
+ per_cpu(cpufreq_cpu_data, j) = policy;
+ add_cpu_dev_symlink(policy, j, get_cpu_device(j));
+ }
+
+ policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
+ GFP_KERNEL);
+ if (!policy->min_freq_req) {
+ ret = -ENOMEM;
+ goto out_destroy_policy;
+ }
+
+ ret = freq_qos_add_request(&policy->constraints,
+ policy->min_freq_req, FREQ_QOS_MIN,
+ FREQ_QOS_MIN_DEFAULT_VALUE);
+ if (ret < 0) {
+ /*
+ * So we don't call freq_qos_remove_request() for an
+ * uninitialized request.
+ */
+ kfree(policy->min_freq_req);
+ policy->min_freq_req = NULL;
+ goto out_destroy_policy;
+ }
+
+ /*
+ * This must be initialized right here to avoid calling
+ * freq_qos_remove_request() on uninitialized request in case
+ * of errors.
+ */
+ policy->max_freq_req = policy->min_freq_req + 1;
+
+ ret = freq_qos_add_request(&policy->constraints,
+ policy->max_freq_req, FREQ_QOS_MAX,
+ FREQ_QOS_MAX_DEFAULT_VALUE);
+ if (ret < 0) {
+ policy->max_freq_req = NULL;
+ goto out_destroy_policy;
+ }
+
+ blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
+ CPUFREQ_CREATE_POLICY, policy);
+ }
+
+ if (cpufreq_driver->get && has_target()) {
+ policy->cur = cpufreq_driver->get(policy->cpu);
+ if (!policy->cur) {
+ ret = -EIO;
+ pr_err("%s: ->get() failed\n", __func__);
+ goto out_destroy_policy;
+ }
+ }
+
+ /*
+ * Sometimes boot loaders set CPU frequency to a value outside of
+ * frequency table present with cpufreq core. In such cases CPU might be
+ * unstable if it has to run on that frequency for long duration of time
+ * and so its better to set it to a frequency which is specified in
+ * freq-table. This also makes cpufreq stats inconsistent as
+ * cpufreq-stats would fail to register because current frequency of CPU
+ * isn't found in freq-table.
+ *
+ * Because we don't want this change to effect boot process badly, we go
+ * for the next freq which is >= policy->cur ('cur' must be set by now,
+ * otherwise we will end up setting freq to lowest of the table as 'cur'
+ * is initialized to zero).
+ *
+ * We are passing target-freq as "policy->cur - 1" otherwise
+ * __cpufreq_driver_target() would simply fail, as policy->cur will be
+ * equal to target-freq.
+ */
+ if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
+ && has_target()) {
+ unsigned int old_freq = policy->cur;
+
+ /* Are we running at unknown frequency ? */
+ ret = cpufreq_frequency_table_get_index(policy, old_freq);
+ if (ret == -EINVAL) {
+ ret = __cpufreq_driver_target(policy, old_freq - 1,
+ CPUFREQ_RELATION_L);
+
+ /*
+ * Reaching here after boot in a few seconds may not
+ * mean that system will remain stable at "unknown"
+ * frequency for longer duration. Hence, a BUG_ON().
+ */
+ BUG_ON(ret);
+ pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n",
+ __func__, policy->cpu, old_freq, policy->cur);
+ }
+ }
+
+ if (new_policy) {
+ ret = cpufreq_add_dev_interface(policy);
+ if (ret)
+ goto out_destroy_policy;
+
+ cpufreq_stats_create_table(policy);
+
+ write_lock_irqsave(&cpufreq_driver_lock, flags);
+ list_add(&policy->policy_list, &cpufreq_policy_list);
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ /*
+ * Register with the energy model before
+ * sched_cpufreq_governor_change() is called, which will result
+ * in rebuilding of the sched domains, which should only be done
+ * once the energy model is properly initialized for the policy
+ * first.
+ *
+ * Also, this should be called before the policy is registered
+ * with cooling framework.
+ */
+ if (cpufreq_driver->register_em)
+ cpufreq_driver->register_em(policy);
+ }
+
+ ret = cpufreq_init_policy(policy);
+ if (ret) {
+ pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
+ __func__, cpu, ret);
+ goto out_destroy_policy;
+ }
+
+ up_write(&policy->rwsem);
+
+ kobject_uevent(&policy->kobj, KOBJ_ADD);
+
+ /* Callback for handling stuff after policy is ready */
+ if (cpufreq_driver->ready)
+ cpufreq_driver->ready(policy);
+
+ if (cpufreq_thermal_control_enabled(cpufreq_driver))
+ policy->cdev = of_cpufreq_cooling_register(policy);
+
+ pr_debug("initialization complete\n");
+
+ return 0;
+
+out_destroy_policy:
+ for_each_cpu(j, policy->real_cpus)
+ remove_cpu_dev_symlink(policy, j, get_cpu_device(j));
+
+out_offline_policy:
+ if (cpufreq_driver->offline)
+ cpufreq_driver->offline(policy);
+
+out_exit_policy:
+ if (cpufreq_driver->exit)
+ cpufreq_driver->exit(policy);
+
+out_free_policy:
+ cpumask_clear(policy->cpus);
+ up_write(&policy->rwsem);
+
+ cpufreq_policy_free(policy);
+ return ret;
+}
+
+/**
+ * cpufreq_add_dev - the cpufreq interface for a CPU device.
+ * @dev: CPU device.
+ * @sif: Subsystem interface structure pointer (not used)
+ */
+static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
+{
+ struct cpufreq_policy *policy;
+ unsigned cpu = dev->id;
+ int ret;
+
+ dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
+
+ if (cpu_online(cpu)) {
+ ret = cpufreq_online(cpu);
+ if (ret)
+ return ret;
+ }
+
+ /* Create sysfs link on CPU registration */
+ policy = per_cpu(cpufreq_cpu_data, cpu);
+ if (policy)
+ add_cpu_dev_symlink(policy, cpu, dev);
+
+ return 0;
+}
+
+static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy)
+{
+ int ret;
+
+ if (has_target())
+ cpufreq_stop_governor(policy);
+
+ cpumask_clear_cpu(cpu, policy->cpus);
+
+ if (!policy_is_inactive(policy)) {
+ /* Nominate a new CPU if necessary. */
+ if (cpu == policy->cpu)
+ policy->cpu = cpumask_any(policy->cpus);
+
+ /* Start the governor again for the active policy. */
+ if (has_target()) {
+ ret = cpufreq_start_governor(policy);
+ if (ret)
+ pr_err("%s: Failed to start governor\n", __func__);
+ }
+
+ return;
+ }
+
+ if (has_target())
+ strncpy(policy->last_governor, policy->governor->name,
+ CPUFREQ_NAME_LEN);
+ else
+ policy->last_policy = policy->policy;
+
+ if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
+ cpufreq_cooling_unregister(policy->cdev);
+ policy->cdev = NULL;
+ }
+
+ if (has_target())
+ cpufreq_exit_governor(policy);
+
+ /*
+ * Perform the ->offline() during light-weight tear-down, as
+ * that allows fast recovery when the CPU comes back.
+ */
+ if (cpufreq_driver->offline) {
+ cpufreq_driver->offline(policy);
+ } else if (cpufreq_driver->exit) {
+ cpufreq_driver->exit(policy);
+ policy->freq_table = NULL;
+ }
+}
+
+static int cpufreq_offline(unsigned int cpu)
+{
+ struct cpufreq_policy *policy;
+
+ pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
+
+ policy = cpufreq_cpu_get_raw(cpu);
+ if (!policy) {
+ pr_debug("%s: No cpu_data found\n", __func__);
+ return 0;
+ }
+
+ down_write(&policy->rwsem);
+
+ __cpufreq_offline(cpu, policy);
+
+ up_write(&policy->rwsem);
+ return 0;
+}
+
+/*
+ * cpufreq_remove_dev - remove a CPU device
+ *
+ * Removes the cpufreq interface for a CPU device.
+ */
+static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
+{
+ unsigned int cpu = dev->id;
+ struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
+
+ if (!policy)
+ return;
+
+ down_write(&policy->rwsem);
+
+ if (cpu_online(cpu))
+ __cpufreq_offline(cpu, policy);
+
+ remove_cpu_dev_symlink(policy, cpu, dev);
+
+ if (!cpumask_empty(policy->real_cpus)) {
+ up_write(&policy->rwsem);
+ return;
+ }
+
+ /* We did light-weight exit earlier, do full tear down now */
+ if (cpufreq_driver->offline)
+ cpufreq_driver->exit(policy);
+
+ up_write(&policy->rwsem);
+
+ cpufreq_policy_free(policy);
+}
+
+/**
+ * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
+ * @policy: Policy managing CPUs.
+ * @new_freq: New CPU frequency.
+ *
+ * Adjust to the current frequency first and clean up later by either calling
+ * cpufreq_update_policy(), or scheduling handle_update().
+ */
+static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
+ unsigned int new_freq)
+{
+ struct cpufreq_freqs freqs;
+
+ pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
+ policy->cur, new_freq);
+
+ freqs.old = policy->cur;
+ freqs.new = new_freq;
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+ cpufreq_freq_transition_end(policy, &freqs, 0);
+}
+
+static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
+{
+ unsigned int new_freq;
+
+ new_freq = cpufreq_driver->get(policy->cpu);
+ if (!new_freq)
+ return 0;
+
+ /*
+ * If fast frequency switching is used with the given policy, the check
+ * against policy->cur is pointless, so skip it in that case.
+ */
+ if (policy->fast_switch_enabled || !has_target())
+ return new_freq;
+
+ if (policy->cur != new_freq) {
+ /*
+ * For some platforms, the frequency returned by hardware may be
+ * slightly different from what is provided in the frequency
+ * table, for example hardware may return 499 MHz instead of 500
+ * MHz. In such cases it is better to avoid getting into
+ * unnecessary frequency updates.
+ */
+ if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
+ return policy->cur;
+
+ cpufreq_out_of_sync(policy, new_freq);
+ if (update)
+ schedule_work(&policy->update);
+ }
+
+ return new_freq;
+}
+
+/**
+ * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
+ * @cpu: CPU number
+ *
+ * This is the last known freq, without actually getting it from the driver.
+ * Return value will be same as what is shown in scaling_cur_freq in sysfs.
+ */
+unsigned int cpufreq_quick_get(unsigned int cpu)
+{
+ struct cpufreq_policy *policy;
+ unsigned int ret_freq = 0;
+ unsigned long flags;
+
+ read_lock_irqsave(&cpufreq_driver_lock, flags);
+
+ if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
+ ret_freq = cpufreq_driver->get(cpu);
+ read_unlock_irqrestore(&cpufreq_driver_lock, flags);
+ return ret_freq;
+ }
+
+ read_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ policy = cpufreq_cpu_get(cpu);
+ if (policy) {
+ ret_freq = policy->cur;
+ cpufreq_cpu_put(policy);
+ }
+
+ return ret_freq;
+}
+EXPORT_SYMBOL(cpufreq_quick_get);
+
+/**
+ * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
+ * @cpu: CPU number
+ *
+ * Just return the max possible frequency for a given CPU.
+ */
+unsigned int cpufreq_quick_get_max(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+ unsigned int ret_freq = 0;
+
+ if (policy) {
+ ret_freq = policy->max;
+ cpufreq_cpu_put(policy);
+ }
+
+ return ret_freq;
+}
+EXPORT_SYMBOL(cpufreq_quick_get_max);
+
+/**
+ * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
+ * @cpu: CPU number
+ *
+ * The default return value is the max_freq field of cpuinfo.
+ */
+__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+ unsigned int ret_freq = 0;
+
+ if (policy) {
+ ret_freq = policy->cpuinfo.max_freq;
+ cpufreq_cpu_put(policy);
+ }
+
+ return ret_freq;
+}
+EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
+
+static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
+{
+ if (unlikely(policy_is_inactive(policy)))
+ return 0;
+
+ return cpufreq_verify_current_freq(policy, true);
+}
+
+/**
+ * cpufreq_get - get the current CPU frequency (in kHz)
+ * @cpu: CPU number
+ *
+ * Get the CPU current (static) CPU frequency
+ */
+unsigned int cpufreq_get(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+ unsigned int ret_freq = 0;
+
+ if (policy) {
+ down_read(&policy->rwsem);
+ if (cpufreq_driver->get)
+ ret_freq = __cpufreq_get(policy);
+ up_read(&policy->rwsem);
+
+ cpufreq_cpu_put(policy);
+ }
+
+ return ret_freq;
+}
+EXPORT_SYMBOL(cpufreq_get);
+
+static struct subsys_interface cpufreq_interface = {
+ .name = "cpufreq",
+ .subsys = &cpu_subsys,
+ .add_dev = cpufreq_add_dev,
+ .remove_dev = cpufreq_remove_dev,
+};
+
+/*
+ * In case platform wants some specific frequency to be configured
+ * during suspend..
+ */
+int cpufreq_generic_suspend(struct cpufreq_policy *policy)
+{
+ int ret;
+
+ if (!policy->suspend_freq) {
+ pr_debug("%s: suspend_freq not defined\n", __func__);
+ return 0;
+ }
+
+ pr_debug("%s: Setting suspend-freq: %u\n", __func__,
+ policy->suspend_freq);
+
+ ret = __cpufreq_driver_target(policy, policy->suspend_freq,
+ CPUFREQ_RELATION_H);
+ if (ret)
+ pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
+ __func__, policy->suspend_freq, ret);
+
+ return ret;
+}
+EXPORT_SYMBOL(cpufreq_generic_suspend);
+
+/**
+ * cpufreq_suspend() - Suspend CPUFreq governors.
+ *
+ * Called during system wide Suspend/Hibernate cycles for suspending governors
+ * as some platforms can't change frequency after this point in suspend cycle.
+ * Because some of the devices (like: i2c, regulators, etc) they use for
+ * changing frequency are suspended quickly after this point.
+ */
+void cpufreq_suspend(void)
+{
+ struct cpufreq_policy *policy;
+
+ if (!cpufreq_driver)
+ return;
+
+ if (!has_target() && !cpufreq_driver->suspend)
+ goto suspend;
+
+ pr_debug("%s: Suspending Governors\n", __func__);
+
+ for_each_active_policy(policy) {
+ if (has_target()) {
+ down_write(&policy->rwsem);
+ cpufreq_stop_governor(policy);
+ up_write(&policy->rwsem);
+ }
+
+ if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
+ pr_err("%s: Failed to suspend driver: %s\n", __func__,
+ cpufreq_driver->name);
+ }
+
+suspend:
+ cpufreq_suspended = true;
+}
+
+/**
+ * cpufreq_resume() - Resume CPUFreq governors.
+ *
+ * Called during system wide Suspend/Hibernate cycle for resuming governors that
+ * are suspended with cpufreq_suspend().
+ */
+void cpufreq_resume(void)
+{
+ struct cpufreq_policy *policy;
+ int ret;
+
+ if (!cpufreq_driver)
+ return;
+
+ if (unlikely(!cpufreq_suspended))
+ return;
+
+ cpufreq_suspended = false;
+
+ if (!has_target() && !cpufreq_driver->resume)
+ return;
+
+ pr_debug("%s: Resuming Governors\n", __func__);
+
+ for_each_active_policy(policy) {
+ if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
+ pr_err("%s: Failed to resume driver: %s\n", __func__,
+ cpufreq_driver->name);
+ } else if (has_target()) {
+ down_write(&policy->rwsem);
+ ret = cpufreq_start_governor(policy);
+ up_write(&policy->rwsem);
+
+ if (ret)
+ pr_err("%s: Failed to start governor for CPU%u's policy\n",
+ __func__, policy->cpu);
+ }
+ }
+}
+
+/**
+ * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
+ * @flags: Flags to test against the current cpufreq driver's flags.
+ *
+ * Assumes that the driver is there, so callers must ensure that this is the
+ * case.
+ */
+bool cpufreq_driver_test_flags(u16 flags)
+{
+ return !!(cpufreq_driver->flags & flags);
+}
+
+/**
+ * cpufreq_get_current_driver - Return the current driver's name.
+ *
+ * Return the name string of the currently registered cpufreq driver or NULL if
+ * none.
+ */
+const char *cpufreq_get_current_driver(void)
+{
+ if (cpufreq_driver)
+ return cpufreq_driver->name;
+
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
+
+/**
+ * cpufreq_get_driver_data - Return current driver data.
+ *
+ * Return the private data of the currently registered cpufreq driver, or NULL
+ * if no cpufreq driver has been registered.
+ */
+void *cpufreq_get_driver_data(void)
+{
+ if (cpufreq_driver)
+ return cpufreq_driver->driver_data;
+
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
+
+/*********************************************************************
+ * NOTIFIER LISTS INTERFACE *
+ *********************************************************************/
+
+/**
+ * cpufreq_register_notifier - Register a notifier with cpufreq.
+ * @nb: notifier function to register.
+ * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
+ *
+ * Add a notifier to one of two lists: either a list of notifiers that run on
+ * clock rate changes (once before and once after every transition), or a list
+ * of notifiers that ron on cpufreq policy changes.
+ *
+ * This function may sleep and it has the same return values as
+ * blocking_notifier_chain_register().
+ */
+int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
+{
+ int ret;
+
+ if (cpufreq_disabled())
+ return -EINVAL;
+
+ switch (list) {
+ case CPUFREQ_TRANSITION_NOTIFIER:
+ mutex_lock(&cpufreq_fast_switch_lock);
+
+ if (cpufreq_fast_switch_count > 0) {
+ mutex_unlock(&cpufreq_fast_switch_lock);
+ return -EBUSY;
+ }
+ ret = srcu_notifier_chain_register(
+ &cpufreq_transition_notifier_list, nb);
+ if (!ret)
+ cpufreq_fast_switch_count--;
+
+ mutex_unlock(&cpufreq_fast_switch_lock);
+ break;
+ case CPUFREQ_POLICY_NOTIFIER:
+ ret = blocking_notifier_chain_register(
+ &cpufreq_policy_notifier_list, nb);
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(cpufreq_register_notifier);
+
+/**
+ * cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
+ * @nb: notifier block to be unregistered.
+ * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
+ *
+ * Remove a notifier from one of the cpufreq notifier lists.
+ *
+ * This function may sleep and it has the same return values as
+ * blocking_notifier_chain_unregister().
+ */
+int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
+{
+ int ret;
+
+ if (cpufreq_disabled())
+ return -EINVAL;
+
+ switch (list) {
+ case CPUFREQ_TRANSITION_NOTIFIER:
+ mutex_lock(&cpufreq_fast_switch_lock);
+
+ ret = srcu_notifier_chain_unregister(
+ &cpufreq_transition_notifier_list, nb);
+ if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
+ cpufreq_fast_switch_count++;
+
+ mutex_unlock(&cpufreq_fast_switch_lock);
+ break;
+ case CPUFREQ_POLICY_NOTIFIER:
+ ret = blocking_notifier_chain_unregister(
+ &cpufreq_policy_notifier_list, nb);
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(cpufreq_unregister_notifier);
+
+
+/*********************************************************************
+ * GOVERNORS *
+ *********************************************************************/
+
+/**
+ * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
+ * @policy: cpufreq policy to switch the frequency for.
+ * @target_freq: New frequency to set (may be approximate).
+ *
+ * Carry out a fast frequency switch without sleeping.
+ *
+ * The driver's ->fast_switch() callback invoked by this function must be
+ * suitable for being called from within RCU-sched read-side critical sections
+ * and it is expected to select the minimum available frequency greater than or
+ * equal to @target_freq (CPUFREQ_RELATION_L).
+ *
+ * This function must not be called if policy->fast_switch_enabled is unset.
+ *
+ * Governors calling this function must guarantee that it will never be invoked
+ * twice in parallel for the same policy and that it will never be called in
+ * parallel with either ->target() or ->target_index() for the same policy.
+ *
+ * Returns the actual frequency set for the CPU.
+ *
+ * If 0 is returned by the driver's ->fast_switch() callback to indicate an
+ * error condition, the hardware configuration must be preserved.
+ */
+unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ unsigned int freq;
+ int cpu;
+
+ target_freq = clamp_val(target_freq, policy->min, policy->max);
+ freq = cpufreq_driver->fast_switch(policy, target_freq);
+
+ if (!freq)
+ return 0;
+
+ policy->cur = freq;
+ arch_set_freq_scale(policy->related_cpus, freq,
+ policy->cpuinfo.max_freq);
+ cpufreq_stats_record_transition(policy, freq);
+
+ if (trace_cpu_frequency_enabled()) {
+ for_each_cpu(cpu, policy->cpus)
+ trace_cpu_frequency(freq, cpu);
+ }
+
+ return freq;
+}
+EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
+
+/**
+ * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
+ * @cpu: Target CPU.
+ * @min_perf: Minimum (required) performance level (units of @capacity).
+ * @target_perf: Target (desired) performance level (units of @capacity).
+ * @capacity: Capacity of the target CPU.
+ *
+ * Carry out a fast performance level switch of @cpu without sleeping.
+ *
+ * The driver's ->adjust_perf() callback invoked by this function must be
+ * suitable for being called from within RCU-sched read-side critical sections
+ * and it is expected to select a suitable performance level equal to or above
+ * @min_perf and preferably equal to or below @target_perf.
+ *
+ * This function must not be called if policy->fast_switch_enabled is unset.
+ *
+ * Governors calling this function must guarantee that it will never be invoked
+ * twice in parallel for the same CPU and that it will never be called in
+ * parallel with either ->target() or ->target_index() or ->fast_switch() for
+ * the same CPU.
+ */
+void cpufreq_driver_adjust_perf(unsigned int cpu,
+ unsigned long min_perf,
+ unsigned long target_perf,
+ unsigned long capacity)
+{
+ cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity);
+}
+
+/**
+ * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
+ *
+ * Return 'true' if the ->adjust_perf callback is present for the
+ * current driver or 'false' otherwise.
+ */
+bool cpufreq_driver_has_adjust_perf(void)
+{
+ return !!cpufreq_driver->adjust_perf;
+}
+
+/* Must set freqs->new to intermediate frequency */
+static int __target_intermediate(struct cpufreq_policy *policy,
+ struct cpufreq_freqs *freqs, int index)
+{
+ int ret;
+
+ freqs->new = cpufreq_driver->get_intermediate(policy, index);
+
+ /* We don't need to switch to intermediate freq */
+ if (!freqs->new)
+ return 0;
+
+ pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
+ __func__, policy->cpu, freqs->old, freqs->new);
+
+ cpufreq_freq_transition_begin(policy, freqs);
+ ret = cpufreq_driver->target_intermediate(policy, index);
+ cpufreq_freq_transition_end(policy, freqs, ret);
+
+ if (ret)
+ pr_err("%s: Failed to change to intermediate frequency: %d\n",
+ __func__, ret);
+
+ return ret;
+}
+
+static int __target_index(struct cpufreq_policy *policy, int index)
+{
+ struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
+ unsigned int restore_freq, intermediate_freq = 0;
+ unsigned int newfreq = policy->freq_table[index].frequency;
+ int retval = -EINVAL;
+ bool notify;
+
+ if (newfreq == policy->cur)
+ return 0;
+
+ /* Save last value to restore later on errors */
+ restore_freq = policy->cur;
+
+ notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
+ if (notify) {
+ /* Handle switching to intermediate frequency */
+ if (cpufreq_driver->get_intermediate) {
+ retval = __target_intermediate(policy, &freqs, index);
+ if (retval)
+ return retval;
+
+ intermediate_freq = freqs.new;
+ /* Set old freq to intermediate */
+ if (intermediate_freq)
+ freqs.old = freqs.new;
+ }
+
+ freqs.new = newfreq;
+ pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
+ __func__, policy->cpu, freqs.old, freqs.new);
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+ }
+
+ retval = cpufreq_driver->target_index(policy, index);
+ if (retval)
+ pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
+ retval);
+
+ if (notify) {
+ cpufreq_freq_transition_end(policy, &freqs, retval);
+
+ /*
+ * Failed after setting to intermediate freq? Driver should have
+ * reverted back to initial frequency and so should we. Check
+ * here for intermediate_freq instead of get_intermediate, in
+ * case we haven't switched to intermediate freq at all.
+ */
+ if (unlikely(retval && intermediate_freq)) {
+ freqs.old = intermediate_freq;
+ freqs.new = restore_freq;
+ cpufreq_freq_transition_begin(policy, &freqs);
+ cpufreq_freq_transition_end(policy, &freqs, 0);
+ }
+ }
+
+ return retval;
+}
+
+int __cpufreq_driver_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ unsigned int old_target_freq = target_freq;
+
+ if (cpufreq_disabled())
+ return -ENODEV;
+
+ target_freq = __resolve_freq(policy, target_freq, relation);
+
+ pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
+ policy->cpu, target_freq, relation, old_target_freq);
+
+ /*
+ * This might look like a redundant call as we are checking it again
+ * after finding index. But it is left intentionally for cases where
+ * exactly same freq is called again and so we can save on few function
+ * calls.
+ */
+ if (target_freq == policy->cur &&
+ !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
+ return 0;
+
+ if (cpufreq_driver->target) {
+ /*
+ * If the driver hasn't setup a single inefficient frequency,
+ * it's unlikely it knows how to decode CPUFREQ_RELATION_E.
+ */
+ if (!policy->efficiencies_available)
+ relation &= ~CPUFREQ_RELATION_E;
+
+ return cpufreq_driver->target(policy, target_freq, relation);
+ }
+
+ if (!cpufreq_driver->target_index)
+ return -EINVAL;
+
+ return __target_index(policy, policy->cached_resolved_idx);
+}
+EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
+
+int cpufreq_driver_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ int ret;
+
+ down_write(&policy->rwsem);
+
+ ret = __cpufreq_driver_target(policy, target_freq, relation);
+
+ up_write(&policy->rwsem);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(cpufreq_driver_target);
+
+__weak struct cpufreq_governor *cpufreq_fallback_governor(void)
+{
+ return NULL;
+}
+
+static int cpufreq_init_governor(struct cpufreq_policy *policy)
+{
+ int ret;
+
+ /* Don't start any governor operations if we are entering suspend */
+ if (cpufreq_suspended)
+ return 0;
+ /*
+ * Governor might not be initiated here if ACPI _PPC changed
+ * notification happened, so check it.
+ */
+ if (!policy->governor)
+ return -EINVAL;
+
+ /* Platform doesn't want dynamic frequency switching ? */
+ if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
+ cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
+ struct cpufreq_governor *gov = cpufreq_fallback_governor();
+
+ if (gov) {
+ pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
+ policy->governor->name, gov->name);
+ policy->governor = gov;
+ } else {
+ return -EINVAL;
+ }
+ }
+
+ if (!try_module_get(policy->governor->owner))
+ return -EINVAL;
+
+ pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
+
+ if (policy->governor->init) {
+ ret = policy->governor->init(policy);
+ if (ret) {
+ module_put(policy->governor->owner);
+ return ret;
+ }
+ }
+
+ policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
+
+ return 0;
+}
+
+static void cpufreq_exit_governor(struct cpufreq_policy *policy)
+{
+ if (cpufreq_suspended || !policy->governor)
+ return;
+
+ pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
+
+ if (policy->governor->exit)
+ policy->governor->exit(policy);
+
+ module_put(policy->governor->owner);
+}
+
+int cpufreq_start_governor(struct cpufreq_policy *policy)
+{
+ int ret;
+
+ if (cpufreq_suspended)
+ return 0;
+
+ if (!policy->governor)
+ return -EINVAL;
+
+ pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
+
+ if (cpufreq_driver->get)
+ cpufreq_verify_current_freq(policy, false);
+
+ if (policy->governor->start) {
+ ret = policy->governor->start(policy);
+ if (ret)
+ return ret;
+ }
+
+ if (policy->governor->limits)
+ policy->governor->limits(policy);
+
+ return 0;
+}
+
+void cpufreq_stop_governor(struct cpufreq_policy *policy)
+{
+ if (cpufreq_suspended || !policy->governor)
+ return;
+
+ pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
+
+ if (policy->governor->stop)
+ policy->governor->stop(policy);
+}
+
+static void cpufreq_governor_limits(struct cpufreq_policy *policy)
+{
+ if (cpufreq_suspended || !policy->governor)
+ return;
+
+ pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
+
+ if (policy->governor->limits)
+ policy->governor->limits(policy);
+}
+
+int cpufreq_register_governor(struct cpufreq_governor *governor)
+{
+ int err;
+
+ if (!governor)
+ return -EINVAL;
+
+ if (cpufreq_disabled())
+ return -ENODEV;
+
+ mutex_lock(&cpufreq_governor_mutex);
+
+ err = -EBUSY;
+ if (!find_governor(governor->name)) {
+ err = 0;
+ list_add(&governor->governor_list, &cpufreq_governor_list);
+ }
+
+ mutex_unlock(&cpufreq_governor_mutex);
+ return err;
+}
+EXPORT_SYMBOL_GPL(cpufreq_register_governor);
+
+void cpufreq_unregister_governor(struct cpufreq_governor *governor)
+{
+ struct cpufreq_policy *policy;
+ unsigned long flags;
+
+ if (!governor)
+ return;
+
+ if (cpufreq_disabled())
+ return;
+
+ /* clear last_governor for all inactive policies */
+ read_lock_irqsave(&cpufreq_driver_lock, flags);
+ for_each_inactive_policy(policy) {
+ if (!strcmp(policy->last_governor, governor->name)) {
+ policy->governor = NULL;
+ strcpy(policy->last_governor, "\0");
+ }
+ }
+ read_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ mutex_lock(&cpufreq_governor_mutex);
+ list_del(&governor->governor_list);
+ mutex_unlock(&cpufreq_governor_mutex);
+}
+EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
+
+
+/*********************************************************************
+ * POLICY INTERFACE *
+ *********************************************************************/
+
+/**
+ * cpufreq_get_policy - get the current cpufreq_policy
+ * @policy: struct cpufreq_policy into which the current cpufreq_policy
+ * is written
+ * @cpu: CPU to find the policy for
+ *
+ * Reads the current cpufreq policy.
+ */
+int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
+{
+ struct cpufreq_policy *cpu_policy;
+ if (!policy)
+ return -EINVAL;
+
+ cpu_policy = cpufreq_cpu_get(cpu);
+ if (!cpu_policy)
+ return -EINVAL;
+
+ memcpy(policy, cpu_policy, sizeof(*policy));
+
+ cpufreq_cpu_put(cpu_policy);
+ return 0;
+}
+EXPORT_SYMBOL(cpufreq_get_policy);
+
+/**
+ * cpufreq_set_policy - Modify cpufreq policy parameters.
+ * @policy: Policy object to modify.
+ * @new_gov: Policy governor pointer.
+ * @new_pol: Policy value (for drivers with built-in governors).
+ *
+ * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
+ * limits to be set for the policy, update @policy with the verified limits
+ * values and either invoke the driver's ->setpolicy() callback (if present) or
+ * carry out a governor update for @policy. That is, run the current governor's
+ * ->limits() callback (if @new_gov points to the same object as the one in
+ * @policy) or replace the governor for @policy with @new_gov.
+ *
+ * The cpuinfo part of @policy is not updated by this function.
+ */
+static int cpufreq_set_policy(struct cpufreq_policy *policy,
+ struct cpufreq_governor *new_gov,
+ unsigned int new_pol)
+{
+ struct cpufreq_policy_data new_data;
+ struct cpufreq_governor *old_gov;
+ int ret;
+
+ memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
+ new_data.freq_table = policy->freq_table;
+ new_data.cpu = policy->cpu;
+ /*
+ * PM QoS framework collects all the requests from users and provide us
+ * the final aggregated value here.
+ */
+ new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
+ new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
+
+ pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
+ new_data.cpu, new_data.min, new_data.max);
+
+ /*
+ * Verify that the CPU speed can be set within these limits and make sure
+ * that min <= max.
+ */
+ ret = cpufreq_driver->verify(&new_data);
+ if (ret)
+ return ret;
+
+ /*
+ * Resolve policy min/max to available frequencies. It ensures
+ * no frequency resolution will neither overshoot the requested maximum
+ * nor undershoot the requested minimum.
+ */
+ policy->min = new_data.min;
+ policy->max = new_data.max;
+ policy->min = __resolve_freq(policy, policy->min, CPUFREQ_RELATION_L);
+ policy->max = __resolve_freq(policy, policy->max, CPUFREQ_RELATION_H);
+ trace_cpu_frequency_limits(policy);
+
+ policy->cached_target_freq = UINT_MAX;
+
+ pr_debug("new min and max freqs are %u - %u kHz\n",
+ policy->min, policy->max);
+
+ if (cpufreq_driver->setpolicy) {
+ policy->policy = new_pol;
+ pr_debug("setting range\n");
+ return cpufreq_driver->setpolicy(policy);
+ }
+
+ if (new_gov == policy->governor) {
+ pr_debug("governor limits update\n");
+ cpufreq_governor_limits(policy);
+ return 0;
+ }
+
+ pr_debug("governor switch\n");
+
+ /* save old, working values */
+ old_gov = policy->governor;
+ /* end old governor */
+ if (old_gov) {
+ cpufreq_stop_governor(policy);
+ cpufreq_exit_governor(policy);
+ }
+
+ /* start new governor */
+ policy->governor = new_gov;
+ ret = cpufreq_init_governor(policy);
+ if (!ret) {
+ ret = cpufreq_start_governor(policy);
+ if (!ret) {
+ pr_debug("governor change\n");
+ sched_cpufreq_governor_change(policy, old_gov);
+ return 0;
+ }
+ cpufreq_exit_governor(policy);
+ }
+
+ /* new governor failed, so re-start old one */
+ pr_debug("starting governor %s failed\n", policy->governor->name);
+ if (old_gov) {
+ policy->governor = old_gov;
+ if (cpufreq_init_governor(policy))
+ policy->governor = NULL;
+ else
+ cpufreq_start_governor(policy);
+ }
+
+ return ret;
+}
+
+/**
+ * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
+ * @cpu: CPU to re-evaluate the policy for.
+ *
+ * Update the current frequency for the cpufreq policy of @cpu and use
+ * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
+ * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
+ * for the policy in question, among other things.
+ */
+void cpufreq_update_policy(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
+
+ if (!policy)
+ return;
+
+ /*
+ * BIOS might change freq behind our back
+ * -> ask driver for current freq and notify governors about a change
+ */
+ if (cpufreq_driver->get && has_target() &&
+ (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
+ goto unlock;
+
+ refresh_frequency_limits(policy);
+
+unlock:
+ cpufreq_cpu_release(policy);
+}
+EXPORT_SYMBOL(cpufreq_update_policy);
+
+/**
+ * cpufreq_update_limits - Update policy limits for a given CPU.
+ * @cpu: CPU to update the policy limits for.
+ *
+ * Invoke the driver's ->update_limits callback if present or call
+ * cpufreq_update_policy() for @cpu.
+ */
+void cpufreq_update_limits(unsigned int cpu)
+{
+ if (cpufreq_driver->update_limits)
+ cpufreq_driver->update_limits(cpu);
+ else
+ cpufreq_update_policy(cpu);
+}
+EXPORT_SYMBOL_GPL(cpufreq_update_limits);
+
+/*********************************************************************
+ * BOOST *
+ *********************************************************************/
+static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
+{
+ int ret;
+
+ if (!policy->freq_table)
+ return -ENXIO;
+
+ ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
+ if (ret) {
+ pr_err("%s: Policy frequency update failed\n", __func__);
+ return ret;
+ }
+
+ ret = freq_qos_update_request(policy->max_freq_req, policy->max);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+int cpufreq_boost_trigger_state(int state)
+{
+ struct cpufreq_policy *policy;
+ unsigned long flags;
+ int ret = 0;
+
+ if (cpufreq_driver->boost_enabled == state)
+ return 0;
+
+ write_lock_irqsave(&cpufreq_driver_lock, flags);
+ cpufreq_driver->boost_enabled = state;
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ cpus_read_lock();
+ for_each_active_policy(policy) {
+ ret = cpufreq_driver->set_boost(policy, state);
+ if (ret)
+ goto err_reset_state;
+
+ policy->boost_enabled = state;
+ }
+ cpus_read_unlock();
+
+ return 0;
+
+err_reset_state:
+ cpus_read_unlock();
+
+ write_lock_irqsave(&cpufreq_driver_lock, flags);
+ cpufreq_driver->boost_enabled = !state;
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ pr_err("%s: Cannot %s BOOST\n",
+ __func__, state ? "enable" : "disable");
+
+ return ret;
+}
+
+static bool cpufreq_boost_supported(void)
+{
+ return cpufreq_driver->set_boost;
+}
+
+static int create_boost_sysfs_file(void)
+{
+ int ret;
+
+ ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
+ if (ret)
+ pr_err("%s: cannot register global BOOST sysfs file\n",
+ __func__);
+
+ return ret;
+}
+
+static void remove_boost_sysfs_file(void)
+{
+ if (cpufreq_boost_supported())
+ sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
+}
+
+int cpufreq_enable_boost_support(void)
+{
+ if (!cpufreq_driver)
+ return -EINVAL;
+
+ if (cpufreq_boost_supported())
+ return 0;
+
+ cpufreq_driver->set_boost = cpufreq_boost_set_sw;
+
+ /* This will get removed on driver unregister */
+ return create_boost_sysfs_file();
+}
+EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
+
+int cpufreq_boost_enabled(void)
+{
+ return cpufreq_driver->boost_enabled;
+}
+EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
+
+/*********************************************************************
+ * REGISTER / UNREGISTER CPUFREQ DRIVER *
+ *********************************************************************/
+static enum cpuhp_state hp_online;
+
+static int cpuhp_cpufreq_online(unsigned int cpu)
+{
+ cpufreq_online(cpu);
+
+ return 0;
+}
+
+static int cpuhp_cpufreq_offline(unsigned int cpu)
+{
+ cpufreq_offline(cpu);
+
+ return 0;
+}
+
+/**
+ * cpufreq_register_driver - register a CPU Frequency driver
+ * @driver_data: A struct cpufreq_driver containing the values#
+ * submitted by the CPU Frequency driver.
+ *
+ * Registers a CPU Frequency driver to this core code. This code
+ * returns zero on success, -EEXIST when another driver got here first
+ * (and isn't unregistered in the meantime).
+ *
+ */
+int cpufreq_register_driver(struct cpufreq_driver *driver_data)
+{
+ unsigned long flags;
+ int ret;
+
+ if (cpufreq_disabled())
+ return -ENODEV;
+
+ /*
+ * The cpufreq core depends heavily on the availability of device
+ * structure, make sure they are available before proceeding further.
+ */
+ if (!get_cpu_device(0))
+ return -EPROBE_DEFER;
+
+ if (!driver_data || !driver_data->verify || !driver_data->init ||
+ !(driver_data->setpolicy || driver_data->target_index ||
+ driver_data->target) ||
+ (driver_data->setpolicy && (driver_data->target_index ||
+ driver_data->target)) ||
+ (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
+ (!driver_data->online != !driver_data->offline) ||
+ (driver_data->adjust_perf && !driver_data->fast_switch))
+ return -EINVAL;
+
+ pr_debug("trying to register driver %s\n", driver_data->name);
+
+ /* Protect against concurrent CPU online/offline. */
+ cpus_read_lock();
+
+ write_lock_irqsave(&cpufreq_driver_lock, flags);
+ if (cpufreq_driver) {
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+ ret = -EEXIST;
+ goto out;
+ }
+ cpufreq_driver = driver_data;
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+ /*
+ * Mark support for the scheduler's frequency invariance engine for
+ * drivers that implement target(), target_index() or fast_switch().
+ */
+ if (!cpufreq_driver->setpolicy) {
+ static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
+ pr_debug("supports frequency invariance");
+ }
+
+ if (driver_data->setpolicy)
+ driver_data->flags |= CPUFREQ_CONST_LOOPS;
+
+ if (cpufreq_boost_supported()) {
+ ret = create_boost_sysfs_file();
+ if (ret)
+ goto err_null_driver;
+ }
+
+ ret = subsys_interface_register(&cpufreq_interface);
+ if (ret)
+ goto err_boost_unreg;
+
+ if (unlikely(list_empty(&cpufreq_policy_list))) {
+ /* if all ->init() calls failed, unregister */
+ ret = -ENODEV;
+ pr_debug("%s: No CPU initialized for driver %s\n", __func__,
+ driver_data->name);
+ goto err_if_unreg;
+ }
+
+ ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
+ "cpufreq:online",
+ cpuhp_cpufreq_online,
+ cpuhp_cpufreq_offline);
+ if (ret < 0)
+ goto err_if_unreg;
+ hp_online = ret;
+ ret = 0;
+
+ pr_debug("driver %s up and running\n", driver_data->name);
+ goto out;
+
+err_if_unreg:
+ subsys_interface_unregister(&cpufreq_interface);
+err_boost_unreg:
+ remove_boost_sysfs_file();
+err_null_driver:
+ write_lock_irqsave(&cpufreq_driver_lock, flags);
+ cpufreq_driver = NULL;
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+out:
+ cpus_read_unlock();
+ return ret;
+}
+EXPORT_SYMBOL_GPL(cpufreq_register_driver);
+
+/*
+ * cpufreq_unregister_driver - unregister the current CPUFreq driver
+ *
+ * Unregister the current CPUFreq driver. Only call this if you have
+ * the right to do so, i.e. if you have succeeded in initialising before!
+ * Returns zero if successful, and -EINVAL if the cpufreq_driver is
+ * currently not initialised.
+ */
+void cpufreq_unregister_driver(struct cpufreq_driver *driver)
+{
+ unsigned long flags;
+
+ if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver)))
+ return;
+
+ pr_debug("unregistering driver %s\n", driver->name);
+
+ /* Protect against concurrent cpu hotplug */
+ cpus_read_lock();
+ subsys_interface_unregister(&cpufreq_interface);
+ remove_boost_sysfs_file();
+ static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
+ cpuhp_remove_state_nocalls_cpuslocked(hp_online);
+
+ write_lock_irqsave(&cpufreq_driver_lock, flags);
+
+ cpufreq_driver = NULL;
+
+ write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+ cpus_read_unlock();
+}
+EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
+
+static int __init cpufreq_core_init(void)
+{
+ struct cpufreq_governor *gov = cpufreq_default_governor();
+ struct device *dev_root;
+
+ if (cpufreq_disabled())
+ return -ENODEV;
+
+ dev_root = bus_get_dev_root(&cpu_subsys);
+ if (dev_root) {
+ cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj);
+ put_device(dev_root);
+ }
+ BUG_ON(!cpufreq_global_kobject);
+
+ if (!strlen(default_governor))
+ strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
+
+ return 0;
+}
+module_param(off, int, 0444);
+module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
+core_initcall(cpufreq_core_init);
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
new file mode 100644
index 0000000000..b6bd0ff353
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * drivers/cpufreq/cpufreq_conservative.c
+ *
+ * Copyright (C) 2001 Russell King
+ * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
+ * Jun Nakajima <jun.nakajima@intel.com>
+ * (C) 2009 Alexander Clouter <alex@digriz.org.uk>
+ */
+
+#include <linux/slab.h>
+#include "cpufreq_governor.h"
+
+struct cs_policy_dbs_info {
+ struct policy_dbs_info policy_dbs;
+ unsigned int down_skip;
+ unsigned int requested_freq;
+};
+
+static inline struct cs_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs)
+{
+ return container_of(policy_dbs, struct cs_policy_dbs_info, policy_dbs);
+}
+
+struct cs_dbs_tuners {
+ unsigned int down_threshold;
+ unsigned int freq_step;
+};
+
+/* Conservative governor macros */
+#define DEF_FREQUENCY_UP_THRESHOLD (80)
+#define DEF_FREQUENCY_DOWN_THRESHOLD (20)
+#define DEF_FREQUENCY_STEP (5)
+#define DEF_SAMPLING_DOWN_FACTOR (1)
+#define MAX_SAMPLING_DOWN_FACTOR (10)
+
+static inline unsigned int get_freq_step(struct cs_dbs_tuners *cs_tuners,
+ struct cpufreq_policy *policy)
+{
+ unsigned int freq_step = (cs_tuners->freq_step * policy->max) / 100;
+
+ /* max freq cannot be less than 100. But who knows... */
+ if (unlikely(freq_step == 0))
+ freq_step = DEF_FREQUENCY_STEP;
+
+ return freq_step;
+}
+
+/*
+ * Every sampling_rate, we check, if current idle time is less than 20%
+ * (default), then we try to increase frequency. Every sampling_rate *
+ * sampling_down_factor, we check, if current idle time is more than 80%
+ * (default), then we try to decrease frequency
+ *
+ * Frequency updates happen at minimum steps of 5% (default) of maximum
+ * frequency
+ */
+static unsigned int cs_dbs_update(struct cpufreq_policy *policy)
+{
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
+ unsigned int requested_freq = dbs_info->requested_freq;
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
+ unsigned int load = dbs_update(policy);
+ unsigned int freq_step;
+
+ /*
+ * break out if we 'cannot' reduce the speed as the user might
+ * want freq_step to be zero
+ */
+ if (cs_tuners->freq_step == 0)
+ goto out;
+
+ /*
+ * If requested_freq is out of range, it is likely that the limits
+ * changed in the meantime, so fall back to current frequency in that
+ * case.
+ */
+ if (requested_freq > policy->max || requested_freq < policy->min) {
+ requested_freq = policy->cur;
+ dbs_info->requested_freq = requested_freq;
+ }
+
+ freq_step = get_freq_step(cs_tuners, policy);
+
+ /*
+ * Decrease requested_freq one freq_step for each idle period that
+ * we didn't update the frequency.
+ */
+ if (policy_dbs->idle_periods < UINT_MAX) {
+ unsigned int freq_steps = policy_dbs->idle_periods * freq_step;
+
+ if (requested_freq > policy->min + freq_steps)
+ requested_freq -= freq_steps;
+ else
+ requested_freq = policy->min;
+
+ policy_dbs->idle_periods = UINT_MAX;
+ }
+
+ /* Check for frequency increase */
+ if (load > dbs_data->up_threshold) {
+ dbs_info->down_skip = 0;
+
+ /* if we are already at full speed then break out early */
+ if (requested_freq == policy->max)
+ goto out;
+
+ requested_freq += freq_step;
+ if (requested_freq > policy->max)
+ requested_freq = policy->max;
+
+ __cpufreq_driver_target(policy, requested_freq,
+ CPUFREQ_RELATION_HE);
+ dbs_info->requested_freq = requested_freq;
+ goto out;
+ }
+
+ /* if sampling_down_factor is active break out early */
+ if (++dbs_info->down_skip < dbs_data->sampling_down_factor)
+ goto out;
+ dbs_info->down_skip = 0;
+
+ /* Check for frequency decrease */
+ if (load < cs_tuners->down_threshold) {
+ /*
+ * if we cannot reduce the frequency anymore, break out early
+ */
+ if (requested_freq == policy->min)
+ goto out;
+
+ if (requested_freq > freq_step)
+ requested_freq -= freq_step;
+ else
+ requested_freq = policy->min;
+
+ __cpufreq_driver_target(policy, requested_freq,
+ CPUFREQ_RELATION_LE);
+ dbs_info->requested_freq = requested_freq;
+ }
+
+ out:
+ return dbs_data->sampling_rate;
+}
+
+/************************** sysfs interface ************************/
+
+static ssize_t sampling_down_factor_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
+ return -EINVAL;
+
+ dbs_data->sampling_down_factor = input;
+ return count;
+}
+
+static ssize_t up_threshold_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1 || input > 100 || input <= cs_tuners->down_threshold)
+ return -EINVAL;
+
+ dbs_data->up_threshold = input;
+ return count;
+}
+
+static ssize_t down_threshold_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ /* cannot be lower than 1 otherwise freq will not fall */
+ if (ret != 1 || input < 1 || input > 100 ||
+ input >= dbs_data->up_threshold)
+ return -EINVAL;
+
+ cs_tuners->down_threshold = input;
+ return count;
+}
+
+static ssize_t ignore_nice_load_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+
+ if (input > 1)
+ input = 1;
+
+ if (input == dbs_data->ignore_nice_load) /* nothing to do */
+ return count;
+
+ dbs_data->ignore_nice_load = input;
+
+ /* we need to re-evaluate prev_cpu_idle */
+ gov_update_cpu_data(dbs_data);
+
+ return count;
+}
+
+static ssize_t freq_step_store(struct gov_attr_set *attr_set, const char *buf,
+ size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1)
+ return -EINVAL;
+
+ if (input > 100)
+ input = 100;
+
+ /*
+ * no need to test here if freq_step is zero as the user might actually
+ * want this, they would be crazy though :)
+ */
+ cs_tuners->freq_step = input;
+ return count;
+}
+
+gov_show_one_common(sampling_rate);
+gov_show_one_common(sampling_down_factor);
+gov_show_one_common(up_threshold);
+gov_show_one_common(ignore_nice_load);
+gov_show_one(cs, down_threshold);
+gov_show_one(cs, freq_step);
+
+gov_attr_rw(sampling_rate);
+gov_attr_rw(sampling_down_factor);
+gov_attr_rw(up_threshold);
+gov_attr_rw(ignore_nice_load);
+gov_attr_rw(down_threshold);
+gov_attr_rw(freq_step);
+
+static struct attribute *cs_attrs[] = {
+ &sampling_rate.attr,
+ &sampling_down_factor.attr,
+ &up_threshold.attr,
+ &down_threshold.attr,
+ &ignore_nice_load.attr,
+ &freq_step.attr,
+ NULL
+};
+ATTRIBUTE_GROUPS(cs);
+
+/************************** sysfs end ************************/
+
+static struct policy_dbs_info *cs_alloc(void)
+{
+ struct cs_policy_dbs_info *dbs_info;
+
+ dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL);
+ return dbs_info ? &dbs_info->policy_dbs : NULL;
+}
+
+static void cs_free(struct policy_dbs_info *policy_dbs)
+{
+ kfree(to_dbs_info(policy_dbs));
+}
+
+static int cs_init(struct dbs_data *dbs_data)
+{
+ struct cs_dbs_tuners *tuners;
+
+ tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
+ if (!tuners)
+ return -ENOMEM;
+
+ tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD;
+ tuners->freq_step = DEF_FREQUENCY_STEP;
+ dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
+ dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
+ dbs_data->ignore_nice_load = 0;
+ dbs_data->tuners = tuners;
+
+ return 0;
+}
+
+static void cs_exit(struct dbs_data *dbs_data)
+{
+ kfree(dbs_data->tuners);
+}
+
+static void cs_start(struct cpufreq_policy *policy)
+{
+ struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
+
+ dbs_info->down_skip = 0;
+ dbs_info->requested_freq = policy->cur;
+}
+
+static struct dbs_governor cs_governor = {
+ .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"),
+ .kobj_type = { .default_groups = cs_groups },
+ .gov_dbs_update = cs_dbs_update,
+ .alloc = cs_alloc,
+ .free = cs_free,
+ .init = cs_init,
+ .exit = cs_exit,
+ .start = cs_start,
+};
+
+#define CPU_FREQ_GOV_CONSERVATIVE (cs_governor.gov)
+
+MODULE_AUTHOR("Alexander Clouter <alex@digriz.org.uk>");
+MODULE_DESCRIPTION("'cpufreq_conservative' - A dynamic cpufreq governor for "
+ "Low Latency Frequency Transition capable processors "
+ "optimised for use in a battery environment");
+MODULE_LICENSE("GPL");
+
+#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE
+struct cpufreq_governor *cpufreq_default_governor(void)
+{
+ return &CPU_FREQ_GOV_CONSERVATIVE;
+}
+#endif
+
+cpufreq_governor_init(CPU_FREQ_GOV_CONSERVATIVE);
+cpufreq_governor_exit(CPU_FREQ_GOV_CONSERVATIVE);
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
new file mode 100644
index 0000000000..af44ee6a64
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -0,0 +1,581 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * drivers/cpufreq/cpufreq_governor.c
+ *
+ * CPUFREQ governors common code
+ *
+ * Copyright (C) 2001 Russell King
+ * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
+ * (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
+ * (C) 2009 Alexander Clouter <alex@digriz.org.uk>
+ * (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/export.h>
+#include <linux/kernel_stat.h>
+#include <linux/slab.h>
+
+#include "cpufreq_governor.h"
+
+#define CPUFREQ_DBS_MIN_SAMPLING_INTERVAL (2 * TICK_NSEC / NSEC_PER_USEC)
+
+static DEFINE_PER_CPU(struct cpu_dbs_info, cpu_dbs);
+
+static DEFINE_MUTEX(gov_dbs_data_mutex);
+
+/* Common sysfs tunables */
+/*
+ * sampling_rate_store - update sampling rate effective immediately if needed.
+ *
+ * If new rate is smaller than the old, simply updating
+ * dbs.sampling_rate might not be appropriate. For example, if the
+ * original sampling_rate was 1 second and the requested new sampling rate is 10
+ * ms because the user needs immediate reaction from ondemand governor, but not
+ * sure if higher frequency will be required or not, then, the governor may
+ * change the sampling rate too late; up to 1 second later. Thus, if we are
+ * reducing the sampling rate, we need to make the new value effective
+ * immediately.
+ *
+ * This must be called with dbs_data->mutex held, otherwise traversing
+ * policy_dbs_list isn't safe.
+ */
+ssize_t sampling_rate_store(struct gov_attr_set *attr_set, const char *buf,
+ size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ struct policy_dbs_info *policy_dbs;
+ unsigned int sampling_interval;
+ int ret;
+
+ ret = sscanf(buf, "%u", &sampling_interval);
+ if (ret != 1 || sampling_interval < CPUFREQ_DBS_MIN_SAMPLING_INTERVAL)
+ return -EINVAL;
+
+ dbs_data->sampling_rate = sampling_interval;
+
+ /*
+ * We are operating under dbs_data->mutex and so the list and its
+ * entries can't be freed concurrently.
+ */
+ list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {
+ mutex_lock(&policy_dbs->update_mutex);
+ /*
+ * On 32-bit architectures this may race with the
+ * sample_delay_ns read in dbs_update_util_handler(), but that
+ * really doesn't matter. If the read returns a value that's
+ * too big, the sample will be skipped, but the next invocation
+ * of dbs_update_util_handler() (when the update has been
+ * completed) will take a sample.
+ *
+ * If this runs in parallel with dbs_work_handler(), we may end
+ * up overwriting the sample_delay_ns value that it has just
+ * written, but it will be corrected next time a sample is
+ * taken, so it shouldn't be significant.
+ */
+ gov_update_sample_delay(policy_dbs, 0);
+ mutex_unlock(&policy_dbs->update_mutex);
+ }
+
+ return count;
+}
+EXPORT_SYMBOL_GPL(sampling_rate_store);
+
+/**
+ * gov_update_cpu_data - Update CPU load data.
+ * @dbs_data: Top-level governor data pointer.
+ *
+ * Update CPU load data for all CPUs in the domain governed by @dbs_data
+ * (that may be a single policy or a bunch of them if governor tunables are
+ * system-wide).
+ *
+ * Call under the @dbs_data mutex.
+ */
+void gov_update_cpu_data(struct dbs_data *dbs_data)
+{
+ struct policy_dbs_info *policy_dbs;
+
+ list_for_each_entry(policy_dbs, &dbs_data->attr_set.policy_list, list) {
+ unsigned int j;
+
+ for_each_cpu(j, policy_dbs->policy->cpus) {
+ struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
+
+ j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time,
+ dbs_data->io_is_busy);
+ if (dbs_data->ignore_nice_load)
+ j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);
+ }
+ }
+}
+EXPORT_SYMBOL_GPL(gov_update_cpu_data);
+
+unsigned int dbs_update(struct cpufreq_policy *policy)
+{
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ unsigned int ignore_nice = dbs_data->ignore_nice_load;
+ unsigned int max_load = 0, idle_periods = UINT_MAX;
+ unsigned int sampling_rate, io_busy, j;
+
+ /*
+ * Sometimes governors may use an additional multiplier to increase
+ * sample delays temporarily. Apply that multiplier to sampling_rate
+ * so as to keep the wake-up-from-idle detection logic a bit
+ * conservative.
+ */
+ sampling_rate = dbs_data->sampling_rate * policy_dbs->rate_mult;
+ /*
+ * For the purpose of ondemand, waiting for disk IO is an indication
+ * that you're performance critical, and not that the system is actually
+ * idle, so do not add the iowait time to the CPU idle time then.
+ */
+ io_busy = dbs_data->io_is_busy;
+
+ /* Get Absolute Load */
+ for_each_cpu(j, policy->cpus) {
+ struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
+ u64 update_time, cur_idle_time;
+ unsigned int idle_time, time_elapsed;
+ unsigned int load;
+
+ cur_idle_time = get_cpu_idle_time(j, &update_time, io_busy);
+
+ time_elapsed = update_time - j_cdbs->prev_update_time;
+ j_cdbs->prev_update_time = update_time;
+
+ idle_time = cur_idle_time - j_cdbs->prev_cpu_idle;
+ j_cdbs->prev_cpu_idle = cur_idle_time;
+
+ if (ignore_nice) {
+ u64 cur_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);
+
+ idle_time += div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC);
+ j_cdbs->prev_cpu_nice = cur_nice;
+ }
+
+ if (unlikely(!time_elapsed)) {
+ /*
+ * That can only happen when this function is called
+ * twice in a row with a very short interval between the
+ * calls, so the previous load value can be used then.
+ */
+ load = j_cdbs->prev_load;
+ } else if (unlikely((int)idle_time > 2 * sampling_rate &&
+ j_cdbs->prev_load)) {
+ /*
+ * If the CPU had gone completely idle and a task has
+ * just woken up on this CPU now, it would be unfair to
+ * calculate 'load' the usual way for this elapsed
+ * time-window, because it would show near-zero load,
+ * irrespective of how CPU intensive that task actually
+ * was. This is undesirable for latency-sensitive bursty
+ * workloads.
+ *
+ * To avoid this, reuse the 'load' from the previous
+ * time-window and give this task a chance to start with
+ * a reasonably high CPU frequency. However, that
+ * shouldn't be over-done, lest we get stuck at a high
+ * load (high frequency) for too long, even when the
+ * current system load has actually dropped down, so
+ * clear prev_load to guarantee that the load will be
+ * computed again next time.
+ *
+ * Detecting this situation is easy: an unusually large
+ * 'idle_time' (as compared to the sampling rate)
+ * indicates this scenario.
+ */
+ load = j_cdbs->prev_load;
+ j_cdbs->prev_load = 0;
+ } else {
+ if (time_elapsed >= idle_time) {
+ load = 100 * (time_elapsed - idle_time) / time_elapsed;
+ } else {
+ /*
+ * That can happen if idle_time is returned by
+ * get_cpu_idle_time_jiffy(). In that case
+ * idle_time is roughly equal to the difference
+ * between time_elapsed and "busy time" obtained
+ * from CPU statistics. Then, the "busy time"
+ * can end up being greater than time_elapsed
+ * (for example, if jiffies_64 and the CPU
+ * statistics are updated by different CPUs),
+ * so idle_time may in fact be negative. That
+ * means, though, that the CPU was busy all
+ * the time (on the rough average) during the
+ * last sampling interval and 100 can be
+ * returned as the load.
+ */
+ load = (int)idle_time < 0 ? 100 : 0;
+ }
+ j_cdbs->prev_load = load;
+ }
+
+ if (unlikely((int)idle_time > 2 * sampling_rate)) {
+ unsigned int periods = idle_time / sampling_rate;
+
+ if (periods < idle_periods)
+ idle_periods = periods;
+ }
+
+ if (load > max_load)
+ max_load = load;
+ }
+
+ policy_dbs->idle_periods = idle_periods;
+
+ return max_load;
+}
+EXPORT_SYMBOL_GPL(dbs_update);
+
+static void dbs_work_handler(struct work_struct *work)
+{
+ struct policy_dbs_info *policy_dbs;
+ struct cpufreq_policy *policy;
+ struct dbs_governor *gov;
+
+ policy_dbs = container_of(work, struct policy_dbs_info, work);
+ policy = policy_dbs->policy;
+ gov = dbs_governor_of(policy);
+
+ /*
+ * Make sure cpufreq_governor_limits() isn't evaluating load or the
+ * ondemand governor isn't updating the sampling rate in parallel.
+ */
+ mutex_lock(&policy_dbs->update_mutex);
+ gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy));
+ mutex_unlock(&policy_dbs->update_mutex);
+
+ /* Allow the utilization update handler to queue up more work. */
+ atomic_set(&policy_dbs->work_count, 0);
+ /*
+ * If the update below is reordered with respect to the sample delay
+ * modification, the utilization update handler may end up using a stale
+ * sample delay value.
+ */
+ smp_wmb();
+ policy_dbs->work_in_progress = false;
+}
+
+static void dbs_irq_work(struct irq_work *irq_work)
+{
+ struct policy_dbs_info *policy_dbs;
+
+ policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work);
+ schedule_work_on(smp_processor_id(), &policy_dbs->work);
+}
+
+static void dbs_update_util_handler(struct update_util_data *data, u64 time,
+ unsigned int flags)
+{
+ struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util);
+ struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
+ u64 delta_ns, lst;
+
+ if (!cpufreq_this_cpu_can_update(policy_dbs->policy))
+ return;
+
+ /*
+ * The work may not be allowed to be queued up right now.
+ * Possible reasons:
+ * - Work has already been queued up or is in progress.
+ * - It is too early (too little time from the previous sample).
+ */
+ if (policy_dbs->work_in_progress)
+ return;
+
+ /*
+ * If the reads below are reordered before the check above, the value
+ * of sample_delay_ns used in the computation may be stale.
+ */
+ smp_rmb();
+ lst = READ_ONCE(policy_dbs->last_sample_time);
+ delta_ns = time - lst;
+ if ((s64)delta_ns < policy_dbs->sample_delay_ns)
+ return;
+
+ /*
+ * If the policy is not shared, the irq_work may be queued up right away
+ * at this point. Otherwise, we need to ensure that only one of the
+ * CPUs sharing the policy will do that.
+ */
+ if (policy_dbs->is_shared) {
+ if (!atomic_add_unless(&policy_dbs->work_count, 1, 1))
+ return;
+
+ /*
+ * If another CPU updated last_sample_time in the meantime, we
+ * shouldn't be here, so clear the work counter and bail out.
+ */
+ if (unlikely(lst != READ_ONCE(policy_dbs->last_sample_time))) {
+ atomic_set(&policy_dbs->work_count, 0);
+ return;
+ }
+ }
+
+ policy_dbs->last_sample_time = time;
+ policy_dbs->work_in_progress = true;
+ irq_work_queue(&policy_dbs->irq_work);
+}
+
+static void gov_set_update_util(struct policy_dbs_info *policy_dbs,
+ unsigned int delay_us)
+{
+ struct cpufreq_policy *policy = policy_dbs->policy;
+ int cpu;
+
+ gov_update_sample_delay(policy_dbs, delay_us);
+ policy_dbs->last_sample_time = 0;
+
+ for_each_cpu(cpu, policy->cpus) {
+ struct cpu_dbs_info *cdbs = &per_cpu(cpu_dbs, cpu);
+
+ cpufreq_add_update_util_hook(cpu, &cdbs->update_util,
+ dbs_update_util_handler);
+ }
+}
+
+static inline void gov_clear_update_util(struct cpufreq_policy *policy)
+{
+ int i;
+
+ for_each_cpu(i, policy->cpus)
+ cpufreq_remove_update_util_hook(i);
+
+ synchronize_rcu();
+}
+
+static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy,
+ struct dbs_governor *gov)
+{
+ struct policy_dbs_info *policy_dbs;
+ int j;
+
+ /* Allocate memory for per-policy governor data. */
+ policy_dbs = gov->alloc();
+ if (!policy_dbs)
+ return NULL;
+
+ policy_dbs->policy = policy;
+ mutex_init(&policy_dbs->update_mutex);
+ atomic_set(&policy_dbs->work_count, 0);
+ init_irq_work(&policy_dbs->irq_work, dbs_irq_work);
+ INIT_WORK(&policy_dbs->work, dbs_work_handler);
+
+ /* Set policy_dbs for all CPUs, online+offline */
+ for_each_cpu(j, policy->related_cpus) {
+ struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
+
+ j_cdbs->policy_dbs = policy_dbs;
+ }
+ return policy_dbs;
+}
+
+static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs,
+ struct dbs_governor *gov)
+{
+ int j;
+
+ mutex_destroy(&policy_dbs->update_mutex);
+
+ for_each_cpu(j, policy_dbs->policy->related_cpus) {
+ struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
+
+ j_cdbs->policy_dbs = NULL;
+ j_cdbs->update_util.func = NULL;
+ }
+ gov->free(policy_dbs);
+}
+
+static void cpufreq_dbs_data_release(struct kobject *kobj)
+{
+ struct dbs_data *dbs_data = to_dbs_data(to_gov_attr_set(kobj));
+ struct dbs_governor *gov = dbs_data->gov;
+
+ gov->exit(dbs_data);
+ kfree(dbs_data);
+}
+
+int cpufreq_dbs_governor_init(struct cpufreq_policy *policy)
+{
+ struct dbs_governor *gov = dbs_governor_of(policy);
+ struct dbs_data *dbs_data;
+ struct policy_dbs_info *policy_dbs;
+ int ret = 0;
+
+ /* State should be equivalent to EXIT */
+ if (policy->governor_data)
+ return -EBUSY;
+
+ policy_dbs = alloc_policy_dbs_info(policy, gov);
+ if (!policy_dbs)
+ return -ENOMEM;
+
+ /* Protect gov->gdbs_data against concurrent updates. */
+ mutex_lock(&gov_dbs_data_mutex);
+
+ dbs_data = gov->gdbs_data;
+ if (dbs_data) {
+ if (WARN_ON(have_governor_per_policy())) {
+ ret = -EINVAL;
+ goto free_policy_dbs_info;
+ }
+ policy_dbs->dbs_data = dbs_data;
+ policy->governor_data = policy_dbs;
+
+ gov_attr_set_get(&dbs_data->attr_set, &policy_dbs->list);
+ goto out;
+ }
+
+ dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
+ if (!dbs_data) {
+ ret = -ENOMEM;
+ goto free_policy_dbs_info;
+ }
+
+ dbs_data->gov = gov;
+ gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list);
+
+ ret = gov->init(dbs_data);
+ if (ret)
+ goto free_dbs_data;
+
+ /*
+ * The sampling interval should not be less than the transition latency
+ * of the CPU and it also cannot be too small for dbs_update() to work
+ * correctly.
+ */
+ dbs_data->sampling_rate = max_t(unsigned int,
+ CPUFREQ_DBS_MIN_SAMPLING_INTERVAL,
+ cpufreq_policy_transition_delay_us(policy));
+
+ if (!have_governor_per_policy())
+ gov->gdbs_data = dbs_data;
+
+ policy_dbs->dbs_data = dbs_data;
+ policy->governor_data = policy_dbs;
+
+ gov->kobj_type.sysfs_ops = &governor_sysfs_ops;
+ gov->kobj_type.release = cpufreq_dbs_data_release;
+ ret = kobject_init_and_add(&dbs_data->attr_set.kobj, &gov->kobj_type,
+ get_governor_parent_kobj(policy),
+ "%s", gov->gov.name);
+ if (!ret)
+ goto out;
+
+ /* Failure, so roll back. */
+ pr_err("initialization failed (dbs_data kobject init error %d)\n", ret);
+
+ kobject_put(&dbs_data->attr_set.kobj);
+
+ policy->governor_data = NULL;
+
+ if (!have_governor_per_policy())
+ gov->gdbs_data = NULL;
+ gov->exit(dbs_data);
+
+free_dbs_data:
+ kfree(dbs_data);
+
+free_policy_dbs_info:
+ free_policy_dbs_info(policy_dbs, gov);
+
+out:
+ mutex_unlock(&gov_dbs_data_mutex);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_init);
+
+void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy)
+{
+ struct dbs_governor *gov = dbs_governor_of(policy);
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ unsigned int count;
+
+ /* Protect gov->gdbs_data against concurrent updates. */
+ mutex_lock(&gov_dbs_data_mutex);
+
+ count = gov_attr_set_put(&dbs_data->attr_set, &policy_dbs->list);
+
+ policy->governor_data = NULL;
+
+ if (!count && !have_governor_per_policy())
+ gov->gdbs_data = NULL;
+
+ free_policy_dbs_info(policy_dbs, gov);
+
+ mutex_unlock(&gov_dbs_data_mutex);
+}
+EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_exit);
+
+int cpufreq_dbs_governor_start(struct cpufreq_policy *policy)
+{
+ struct dbs_governor *gov = dbs_governor_of(policy);
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ unsigned int sampling_rate, ignore_nice, j;
+ unsigned int io_busy;
+
+ if (!policy->cur)
+ return -EINVAL;
+
+ policy_dbs->is_shared = policy_is_shared(policy);
+ policy_dbs->rate_mult = 1;
+
+ sampling_rate = dbs_data->sampling_rate;
+ ignore_nice = dbs_data->ignore_nice_load;
+ io_busy = dbs_data->io_is_busy;
+
+ for_each_cpu(j, policy->cpus) {
+ struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
+
+ j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time, io_busy);
+ /*
+ * Make the first invocation of dbs_update() compute the load.
+ */
+ j_cdbs->prev_load = 0;
+
+ if (ignore_nice)
+ j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);
+ }
+
+ gov->start(policy);
+
+ gov_set_update_util(policy_dbs, sampling_rate);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_start);
+
+void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy)
+{
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+
+ gov_clear_update_util(policy_dbs->policy);
+ irq_work_sync(&policy_dbs->irq_work);
+ cancel_work_sync(&policy_dbs->work);
+ atomic_set(&policy_dbs->work_count, 0);
+ policy_dbs->work_in_progress = false;
+}
+EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop);
+
+void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy)
+{
+ struct policy_dbs_info *policy_dbs;
+
+ /* Protect gov->gdbs_data against cpufreq_dbs_governor_exit() */
+ mutex_lock(&gov_dbs_data_mutex);
+ policy_dbs = policy->governor_data;
+ if (!policy_dbs)
+ goto out;
+
+ mutex_lock(&policy_dbs->update_mutex);
+ cpufreq_policy_apply_limits(policy);
+ gov_update_sample_delay(policy_dbs, 0);
+ mutex_unlock(&policy_dbs->update_mutex);
+
+out:
+ mutex_unlock(&gov_dbs_data_mutex);
+}
+EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits);
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
new file mode 100644
index 0000000000..168c23fd7f
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -0,0 +1,181 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * drivers/cpufreq/cpufreq_governor.h
+ *
+ * Header file for CPUFreq governors common code
+ *
+ * Copyright (C) 2001 Russell King
+ * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
+ * (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
+ * (C) 2009 Alexander Clouter <alex@digriz.org.uk>
+ * (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
+ */
+
+#ifndef _CPUFREQ_GOVERNOR_H
+#define _CPUFREQ_GOVERNOR_H
+
+#include <linux/atomic.h>
+#include <linux/irq_work.h>
+#include <linux/cpufreq.h>
+#include <linux/sched/cpufreq.h>
+#include <linux/kernel_stat.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+
+/* Ondemand Sampling types */
+enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE};
+
+/*
+ * Abbreviations:
+ * dbs: used as a shortform for demand based switching It helps to keep variable
+ * names smaller, simpler
+ * cdbs: common dbs
+ * od_*: On-demand governor
+ * cs_*: Conservative governor
+ */
+
+/* Governor demand based switching data (per-policy or global). */
+struct dbs_data {
+ struct gov_attr_set attr_set;
+ struct dbs_governor *gov;
+ void *tuners;
+ unsigned int ignore_nice_load;
+ unsigned int sampling_rate;
+ unsigned int sampling_down_factor;
+ unsigned int up_threshold;
+ unsigned int io_is_busy;
+};
+
+static inline struct dbs_data *to_dbs_data(struct gov_attr_set *attr_set)
+{
+ return container_of(attr_set, struct dbs_data, attr_set);
+}
+
+#define gov_show_one(_gov, file_name) \
+static ssize_t file_name##_show \
+(struct gov_attr_set *attr_set, char *buf) \
+{ \
+ struct dbs_data *dbs_data = to_dbs_data(attr_set); \
+ struct _gov##_dbs_tuners *tuners = dbs_data->tuners; \
+ return sprintf(buf, "%u\n", tuners->file_name); \
+}
+
+#define gov_show_one_common(file_name) \
+static ssize_t file_name##_show \
+(struct gov_attr_set *attr_set, char *buf) \
+{ \
+ struct dbs_data *dbs_data = to_dbs_data(attr_set); \
+ return sprintf(buf, "%u\n", dbs_data->file_name); \
+}
+
+#define gov_attr_ro(_name) \
+static struct governor_attr _name = __ATTR_RO(_name)
+
+#define gov_attr_rw(_name) \
+static struct governor_attr _name = __ATTR_RW(_name)
+
+/* Common to all CPUs of a policy */
+struct policy_dbs_info {
+ struct cpufreq_policy *policy;
+ /*
+ * Per policy mutex that serializes load evaluation from limit-change
+ * and work-handler.
+ */
+ struct mutex update_mutex;
+
+ u64 last_sample_time;
+ s64 sample_delay_ns;
+ atomic_t work_count;
+ struct irq_work irq_work;
+ struct work_struct work;
+ /* dbs_data may be shared between multiple policy objects */
+ struct dbs_data *dbs_data;
+ struct list_head list;
+ /* Multiplier for increasing sample delay temporarily. */
+ unsigned int rate_mult;
+ unsigned int idle_periods; /* For conservative */
+ /* Status indicators */
+ bool is_shared; /* This object is used by multiple CPUs */
+ bool work_in_progress; /* Work is being queued up or in progress */
+};
+
+static inline void gov_update_sample_delay(struct policy_dbs_info *policy_dbs,
+ unsigned int delay_us)
+{
+ policy_dbs->sample_delay_ns = delay_us * NSEC_PER_USEC;
+}
+
+/* Per cpu structures */
+struct cpu_dbs_info {
+ u64 prev_cpu_idle;
+ u64 prev_update_time;
+ u64 prev_cpu_nice;
+ /*
+ * Used to keep track of load in the previous interval. However, when
+ * explicitly set to zero, it is used as a flag to ensure that we copy
+ * the previous load to the current interval only once, upon the first
+ * wake-up from idle.
+ */
+ unsigned int prev_load;
+ struct update_util_data update_util;
+ struct policy_dbs_info *policy_dbs;
+};
+
+/* Common Governor data across policies */
+struct dbs_governor {
+ struct cpufreq_governor gov;
+ struct kobj_type kobj_type;
+
+ /*
+ * Common data for platforms that don't set
+ * CPUFREQ_HAVE_GOVERNOR_PER_POLICY
+ */
+ struct dbs_data *gdbs_data;
+
+ unsigned int (*gov_dbs_update)(struct cpufreq_policy *policy);
+ struct policy_dbs_info *(*alloc)(void);
+ void (*free)(struct policy_dbs_info *policy_dbs);
+ int (*init)(struct dbs_data *dbs_data);
+ void (*exit)(struct dbs_data *dbs_data);
+ void (*start)(struct cpufreq_policy *policy);
+};
+
+static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy)
+{
+ return container_of(policy->governor, struct dbs_governor, gov);
+}
+
+/* Governor callback routines */
+int cpufreq_dbs_governor_init(struct cpufreq_policy *policy);
+void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy);
+int cpufreq_dbs_governor_start(struct cpufreq_policy *policy);
+void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy);
+void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy);
+
+#define CPUFREQ_DBS_GOVERNOR_INITIALIZER(_name_) \
+ { \
+ .name = _name_, \
+ .flags = CPUFREQ_GOV_DYNAMIC_SWITCHING, \
+ .owner = THIS_MODULE, \
+ .init = cpufreq_dbs_governor_init, \
+ .exit = cpufreq_dbs_governor_exit, \
+ .start = cpufreq_dbs_governor_start, \
+ .stop = cpufreq_dbs_governor_stop, \
+ .limits = cpufreq_dbs_governor_limits, \
+ }
+
+/* Governor specific operations */
+struct od_ops {
+ unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy,
+ unsigned int freq_next, unsigned int relation);
+};
+
+unsigned int dbs_update(struct cpufreq_policy *policy);
+void od_register_powersave_bias_handler(unsigned int (*f)
+ (struct cpufreq_policy *, unsigned int, unsigned int),
+ unsigned int powersave_bias);
+void od_unregister_powersave_bias_handler(void);
+ssize_t sampling_rate_store(struct gov_attr_set *attr_set, const char *buf,
+ size_t count);
+void gov_update_cpu_data(struct dbs_data *dbs_data);
+#endif /* _CPUFREQ_GOVERNOR_H */
diff --git a/drivers/cpufreq/cpufreq_governor_attr_set.c b/drivers/cpufreq/cpufreq_governor_attr_set.c
new file mode 100644
index 0000000000..771770ea0e
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_governor_attr_set.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Abstract code for CPUFreq governor tunable sysfs attributes.
+ *
+ * Copyright (C) 2016, Intel Corporation
+ * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
+ */
+
+#include "cpufreq_governor.h"
+
+static inline struct governor_attr *to_gov_attr(struct attribute *attr)
+{
+ return container_of(attr, struct governor_attr, attr);
+}
+
+static ssize_t governor_show(struct kobject *kobj, struct attribute *attr,
+ char *buf)
+{
+ struct governor_attr *gattr = to_gov_attr(attr);
+
+ return gattr->show(to_gov_attr_set(kobj), buf);
+}
+
+static ssize_t governor_store(struct kobject *kobj, struct attribute *attr,
+ const char *buf, size_t count)
+{
+ struct gov_attr_set *attr_set = to_gov_attr_set(kobj);
+ struct governor_attr *gattr = to_gov_attr(attr);
+ int ret;
+
+ mutex_lock(&attr_set->update_lock);
+ ret = attr_set->usage_count ? gattr->store(attr_set, buf, count) : -EBUSY;
+ mutex_unlock(&attr_set->update_lock);
+ return ret;
+}
+
+const struct sysfs_ops governor_sysfs_ops = {
+ .show = governor_show,
+ .store = governor_store,
+};
+EXPORT_SYMBOL_GPL(governor_sysfs_ops);
+
+void gov_attr_set_init(struct gov_attr_set *attr_set, struct list_head *list_node)
+{
+ INIT_LIST_HEAD(&attr_set->policy_list);
+ mutex_init(&attr_set->update_lock);
+ attr_set->usage_count = 1;
+ list_add(list_node, &attr_set->policy_list);
+}
+EXPORT_SYMBOL_GPL(gov_attr_set_init);
+
+void gov_attr_set_get(struct gov_attr_set *attr_set, struct list_head *list_node)
+{
+ mutex_lock(&attr_set->update_lock);
+ attr_set->usage_count++;
+ list_add(list_node, &attr_set->policy_list);
+ mutex_unlock(&attr_set->update_lock);
+}
+EXPORT_SYMBOL_GPL(gov_attr_set_get);
+
+unsigned int gov_attr_set_put(struct gov_attr_set *attr_set, struct list_head *list_node)
+{
+ unsigned int count;
+
+ mutex_lock(&attr_set->update_lock);
+ list_del(list_node);
+ count = --attr_set->usage_count;
+ mutex_unlock(&attr_set->update_lock);
+ if (count)
+ return count;
+
+ mutex_destroy(&attr_set->update_lock);
+ kobject_put(&attr_set->kobj);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(gov_attr_set_put);
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
new file mode 100644
index 0000000000..c52d19d675
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -0,0 +1,486 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * drivers/cpufreq/cpufreq_ondemand.c
+ *
+ * Copyright (C) 2001 Russell King
+ * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
+ * Jun Nakajima <jun.nakajima@intel.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpu.h>
+#include <linux/percpu-defs.h>
+#include <linux/slab.h>
+#include <linux/tick.h>
+#include <linux/sched/cpufreq.h>
+
+#include "cpufreq_ondemand.h"
+
+/* On-demand governor macros */
+#define DEF_FREQUENCY_UP_THRESHOLD (80)
+#define DEF_SAMPLING_DOWN_FACTOR (1)
+#define MAX_SAMPLING_DOWN_FACTOR (100000)
+#define MICRO_FREQUENCY_UP_THRESHOLD (95)
+#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000)
+#define MIN_FREQUENCY_UP_THRESHOLD (1)
+#define MAX_FREQUENCY_UP_THRESHOLD (100)
+
+static struct od_ops od_ops;
+
+static unsigned int default_powersave_bias;
+
+/*
+ * Not all CPUs want IO time to be accounted as busy; this depends on how
+ * efficient idling at a higher frequency/voltage is.
+ * Pavel Machek says this is not so for various generations of AMD and old
+ * Intel systems.
+ * Mike Chan (android.com) claims this is also not true for ARM.
+ * Because of this, whitelist specific known (series) of CPUs by default, and
+ * leave all others up to the user.
+ */
+static int should_io_be_busy(void)
+{
+#if defined(CONFIG_X86)
+ /*
+ * For Intel, Core 2 (model 15) and later have an efficient idle.
+ */
+ if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
+ boot_cpu_data.x86 == 6 &&
+ boot_cpu_data.x86_model >= 15)
+ return 1;
+#endif
+ return 0;
+}
+
+/*
+ * Find right freq to be set now with powersave_bias on.
+ * Returns the freq_hi to be used right now and will set freq_hi_delay_us,
+ * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs.
+ */
+static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
+ unsigned int freq_next, unsigned int relation)
+{
+ unsigned int freq_req, freq_reduc, freq_avg;
+ unsigned int freq_hi, freq_lo;
+ unsigned int index;
+ unsigned int delay_hi_us;
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ struct od_dbs_tuners *od_tuners = dbs_data->tuners;
+ struct cpufreq_frequency_table *freq_table = policy->freq_table;
+
+ if (!freq_table) {
+ dbs_info->freq_lo = 0;
+ dbs_info->freq_lo_delay_us = 0;
+ return freq_next;
+ }
+
+ index = cpufreq_frequency_table_target(policy, freq_next, relation);
+ freq_req = freq_table[index].frequency;
+ freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
+ freq_avg = freq_req - freq_reduc;
+
+ /* Find freq bounds for freq_avg in freq_table */
+ index = cpufreq_table_find_index_h(policy, freq_avg,
+ relation & CPUFREQ_RELATION_E);
+ freq_lo = freq_table[index].frequency;
+ index = cpufreq_table_find_index_l(policy, freq_avg,
+ relation & CPUFREQ_RELATION_E);
+ freq_hi = freq_table[index].frequency;
+
+ /* Find out how long we have to be in hi and lo freqs */
+ if (freq_hi == freq_lo) {
+ dbs_info->freq_lo = 0;
+ dbs_info->freq_lo_delay_us = 0;
+ return freq_lo;
+ }
+ delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate;
+ delay_hi_us += (freq_hi - freq_lo) / 2;
+ delay_hi_us /= freq_hi - freq_lo;
+ dbs_info->freq_hi_delay_us = delay_hi_us;
+ dbs_info->freq_lo = freq_lo;
+ dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us;
+ return freq_hi;
+}
+
+static void ondemand_powersave_bias_init(struct cpufreq_policy *policy)
+{
+ struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
+
+ dbs_info->freq_lo = 0;
+}
+
+static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
+{
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ struct od_dbs_tuners *od_tuners = dbs_data->tuners;
+
+ if (od_tuners->powersave_bias)
+ freq = od_ops.powersave_bias_target(policy, freq,
+ CPUFREQ_RELATION_HE);
+ else if (policy->cur == policy->max)
+ return;
+
+ __cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ?
+ CPUFREQ_RELATION_LE : CPUFREQ_RELATION_HE);
+}
+
+/*
+ * Every sampling_rate, we check, if current idle time is less than 20%
+ * (default), then we try to increase frequency. Else, we adjust the frequency
+ * proportional to load.
+ */
+static void od_update(struct cpufreq_policy *policy)
+{
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ struct od_dbs_tuners *od_tuners = dbs_data->tuners;
+ unsigned int load = dbs_update(policy);
+
+ dbs_info->freq_lo = 0;
+
+ /* Check for frequency increase */
+ if (load > dbs_data->up_threshold) {
+ /* If switching to max speed, apply sampling_down_factor */
+ if (policy->cur < policy->max)
+ policy_dbs->rate_mult = dbs_data->sampling_down_factor;
+ dbs_freq_increase(policy, policy->max);
+ } else {
+ /* Calculate the next frequency proportional to load */
+ unsigned int freq_next, min_f, max_f;
+
+ min_f = policy->cpuinfo.min_freq;
+ max_f = policy->cpuinfo.max_freq;
+ freq_next = min_f + load * (max_f - min_f) / 100;
+
+ /* No longer fully busy, reset rate_mult */
+ policy_dbs->rate_mult = 1;
+
+ if (od_tuners->powersave_bias)
+ freq_next = od_ops.powersave_bias_target(policy,
+ freq_next,
+ CPUFREQ_RELATION_LE);
+
+ __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_CE);
+ }
+}
+
+static unsigned int od_dbs_update(struct cpufreq_policy *policy)
+{
+ struct policy_dbs_info *policy_dbs = policy->governor_data;
+ struct dbs_data *dbs_data = policy_dbs->dbs_data;
+ struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
+ int sample_type = dbs_info->sample_type;
+
+ /* Common NORMAL_SAMPLE setup */
+ dbs_info->sample_type = OD_NORMAL_SAMPLE;
+ /*
+ * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore
+ * it then.
+ */
+ if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) {
+ __cpufreq_driver_target(policy, dbs_info->freq_lo,
+ CPUFREQ_RELATION_HE);
+ return dbs_info->freq_lo_delay_us;
+ }
+
+ od_update(policy);
+
+ if (dbs_info->freq_lo) {
+ /* Setup SUB_SAMPLE */
+ dbs_info->sample_type = OD_SUB_SAMPLE;
+ return dbs_info->freq_hi_delay_us;
+ }
+
+ return dbs_data->sampling_rate * policy_dbs->rate_mult;
+}
+
+/************************** sysfs interface ************************/
+static struct dbs_governor od_dbs_gov;
+
+static ssize_t io_is_busy_store(struct gov_attr_set *attr_set, const char *buf,
+ size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+ dbs_data->io_is_busy = !!input;
+
+ /* we need to re-evaluate prev_cpu_idle */
+ gov_update_cpu_data(dbs_data);
+
+ return count;
+}
+
+static ssize_t up_threshold_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
+ input < MIN_FREQUENCY_UP_THRESHOLD) {
+ return -EINVAL;
+ }
+
+ dbs_data->up_threshold = input;
+ return count;
+}
+
+static ssize_t sampling_down_factor_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ struct policy_dbs_info *policy_dbs;
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
+ return -EINVAL;
+
+ dbs_data->sampling_down_factor = input;
+
+ /* Reset down sampling multiplier in case it was active */
+ list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {
+ /*
+ * Doing this without locking might lead to using different
+ * rate_mult values in od_update() and od_dbs_update().
+ */
+ mutex_lock(&policy_dbs->update_mutex);
+ policy_dbs->rate_mult = 1;
+ mutex_unlock(&policy_dbs->update_mutex);
+ }
+
+ return count;
+}
+
+static ssize_t ignore_nice_load_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+
+ if (input > 1)
+ input = 1;
+
+ if (input == dbs_data->ignore_nice_load) { /* nothing to do */
+ return count;
+ }
+ dbs_data->ignore_nice_load = input;
+
+ /* we need to re-evaluate prev_cpu_idle */
+ gov_update_cpu_data(dbs_data);
+
+ return count;
+}
+
+static ssize_t powersave_bias_store(struct gov_attr_set *attr_set,
+ const char *buf, size_t count)
+{
+ struct dbs_data *dbs_data = to_dbs_data(attr_set);
+ struct od_dbs_tuners *od_tuners = dbs_data->tuners;
+ struct policy_dbs_info *policy_dbs;
+ unsigned int input;
+ int ret;
+ ret = sscanf(buf, "%u", &input);
+
+ if (ret != 1)
+ return -EINVAL;
+
+ if (input > 1000)
+ input = 1000;
+
+ od_tuners->powersave_bias = input;
+
+ list_for_each_entry(policy_dbs, &attr_set->policy_list, list)
+ ondemand_powersave_bias_init(policy_dbs->policy);
+
+ return count;
+}
+
+gov_show_one_common(sampling_rate);
+gov_show_one_common(up_threshold);
+gov_show_one_common(sampling_down_factor);
+gov_show_one_common(ignore_nice_load);
+gov_show_one_common(io_is_busy);
+gov_show_one(od, powersave_bias);
+
+gov_attr_rw(sampling_rate);
+gov_attr_rw(io_is_busy);
+gov_attr_rw(up_threshold);
+gov_attr_rw(sampling_down_factor);
+gov_attr_rw(ignore_nice_load);
+gov_attr_rw(powersave_bias);
+
+static struct attribute *od_attrs[] = {
+ &sampling_rate.attr,
+ &up_threshold.attr,
+ &sampling_down_factor.attr,
+ &ignore_nice_load.attr,
+ &powersave_bias.attr,
+ &io_is_busy.attr,
+ NULL
+};
+ATTRIBUTE_GROUPS(od);
+
+/************************** sysfs end ************************/
+
+static struct policy_dbs_info *od_alloc(void)
+{
+ struct od_policy_dbs_info *dbs_info;
+
+ dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL);
+ return dbs_info ? &dbs_info->policy_dbs : NULL;
+}
+
+static void od_free(struct policy_dbs_info *policy_dbs)
+{
+ kfree(to_dbs_info(policy_dbs));
+}
+
+static int od_init(struct dbs_data *dbs_data)
+{
+ struct od_dbs_tuners *tuners;
+ u64 idle_time;
+ int cpu;
+
+ tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
+ if (!tuners)
+ return -ENOMEM;
+
+ cpu = get_cpu();
+ idle_time = get_cpu_idle_time_us(cpu, NULL);
+ put_cpu();
+ if (idle_time != -1ULL) {
+ /* Idle micro accounting is supported. Use finer thresholds */
+ dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
+ } else {
+ dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
+ }
+
+ dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
+ dbs_data->ignore_nice_load = 0;
+ tuners->powersave_bias = default_powersave_bias;
+ dbs_data->io_is_busy = should_io_be_busy();
+
+ dbs_data->tuners = tuners;
+ return 0;
+}
+
+static void od_exit(struct dbs_data *dbs_data)
+{
+ kfree(dbs_data->tuners);
+}
+
+static void od_start(struct cpufreq_policy *policy)
+{
+ struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
+
+ dbs_info->sample_type = OD_NORMAL_SAMPLE;
+ ondemand_powersave_bias_init(policy);
+}
+
+static struct od_ops od_ops = {
+ .powersave_bias_target = generic_powersave_bias_target,
+};
+
+static struct dbs_governor od_dbs_gov = {
+ .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"),
+ .kobj_type = { .default_groups = od_groups },
+ .gov_dbs_update = od_dbs_update,
+ .alloc = od_alloc,
+ .free = od_free,
+ .init = od_init,
+ .exit = od_exit,
+ .start = od_start,
+};
+
+#define CPU_FREQ_GOV_ONDEMAND (od_dbs_gov.gov)
+
+static void od_set_powersave_bias(unsigned int powersave_bias)
+{
+ unsigned int cpu;
+ cpumask_var_t done;
+
+ if (!alloc_cpumask_var(&done, GFP_KERNEL))
+ return;
+
+ default_powersave_bias = powersave_bias;
+ cpumask_clear(done);
+
+ cpus_read_lock();
+ for_each_online_cpu(cpu) {
+ struct cpufreq_policy *policy;
+ struct policy_dbs_info *policy_dbs;
+ struct dbs_data *dbs_data;
+ struct od_dbs_tuners *od_tuners;
+
+ if (cpumask_test_cpu(cpu, done))
+ continue;
+
+ policy = cpufreq_cpu_get_raw(cpu);
+ if (!policy || policy->governor != &CPU_FREQ_GOV_ONDEMAND)
+ continue;
+
+ policy_dbs = policy->governor_data;
+ if (!policy_dbs)
+ continue;
+
+ cpumask_or(done, done, policy->cpus);
+
+ dbs_data = policy_dbs->dbs_data;
+ od_tuners = dbs_data->tuners;
+ od_tuners->powersave_bias = default_powersave_bias;
+ }
+ cpus_read_unlock();
+
+ free_cpumask_var(done);
+}
+
+void od_register_powersave_bias_handler(unsigned int (*f)
+ (struct cpufreq_policy *, unsigned int, unsigned int),
+ unsigned int powersave_bias)
+{
+ od_ops.powersave_bias_target = f;
+ od_set_powersave_bias(powersave_bias);
+}
+EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler);
+
+void od_unregister_powersave_bias_handler(void)
+{
+ od_ops.powersave_bias_target = generic_powersave_bias_target;
+ od_set_powersave_bias(0);
+}
+EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler);
+
+MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
+MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
+MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
+ "Low Latency Frequency Transition capable processors");
+MODULE_LICENSE("GPL");
+
+#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
+struct cpufreq_governor *cpufreq_default_governor(void)
+{
+ return &CPU_FREQ_GOV_ONDEMAND;
+}
+#endif
+
+cpufreq_governor_init(CPU_FREQ_GOV_ONDEMAND);
+cpufreq_governor_exit(CPU_FREQ_GOV_ONDEMAND);
diff --git a/drivers/cpufreq/cpufreq_ondemand.h b/drivers/cpufreq/cpufreq_ondemand.h
new file mode 100644
index 0000000000..1af8e5c4b8
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_ondemand.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Header file for CPUFreq ondemand governor and related code.
+ *
+ * Copyright (C) 2016, Intel Corporation
+ * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
+ */
+
+#include "cpufreq_governor.h"
+
+struct od_policy_dbs_info {
+ struct policy_dbs_info policy_dbs;
+ unsigned int freq_lo;
+ unsigned int freq_lo_delay_us;
+ unsigned int freq_hi_delay_us;
+ unsigned int sample_type:1;
+};
+
+static inline struct od_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs)
+{
+ return container_of(policy_dbs, struct od_policy_dbs_info, policy_dbs);
+}
+
+struct od_dbs_tuners {
+ unsigned int powersave_bias;
+};
diff --git a/drivers/cpufreq/cpufreq_performance.c b/drivers/cpufreq/cpufreq_performance.c
new file mode 100644
index 0000000000..addd93f2a4
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_performance.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/drivers/cpufreq/cpufreq_performance.c
+ *
+ * Copyright (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/module.h>
+
+static void cpufreq_gov_performance_limits(struct cpufreq_policy *policy)
+{
+ pr_debug("setting to %u kHz\n", policy->max);
+ __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
+}
+
+static struct cpufreq_governor cpufreq_gov_performance = {
+ .name = "performance",
+ .owner = THIS_MODULE,
+ .flags = CPUFREQ_GOV_STRICT_TARGET,
+ .limits = cpufreq_gov_performance_limits,
+};
+
+#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
+struct cpufreq_governor *cpufreq_default_governor(void)
+{
+ return &cpufreq_gov_performance;
+}
+#endif
+#ifndef CONFIG_CPU_FREQ_GOV_PERFORMANCE_MODULE
+struct cpufreq_governor *cpufreq_fallback_governor(void)
+{
+ return &cpufreq_gov_performance;
+}
+#endif
+
+MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>");
+MODULE_DESCRIPTION("CPUfreq policy governor 'performance'");
+MODULE_LICENSE("GPL");
+
+cpufreq_governor_init(cpufreq_gov_performance);
+cpufreq_governor_exit(cpufreq_gov_performance);
diff --git a/drivers/cpufreq/cpufreq_powersave.c b/drivers/cpufreq/cpufreq_powersave.c
new file mode 100644
index 0000000000..8d830d860e
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_powersave.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/drivers/cpufreq/cpufreq_powersave.c
+ *
+ * Copyright (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/module.h>
+
+static void cpufreq_gov_powersave_limits(struct cpufreq_policy *policy)
+{
+ pr_debug("setting to %u kHz\n", policy->min);
+ __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
+}
+
+static struct cpufreq_governor cpufreq_gov_powersave = {
+ .name = "powersave",
+ .limits = cpufreq_gov_powersave_limits,
+ .owner = THIS_MODULE,
+ .flags = CPUFREQ_GOV_STRICT_TARGET,
+};
+
+MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>");
+MODULE_DESCRIPTION("CPUfreq policy governor 'powersave'");
+MODULE_LICENSE("GPL");
+
+#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE
+struct cpufreq_governor *cpufreq_default_governor(void)
+{
+ return &cpufreq_gov_powersave;
+}
+#endif
+
+cpufreq_governor_init(cpufreq_gov_powersave);
+cpufreq_governor_exit(cpufreq_gov_powersave);
diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c
new file mode 100644
index 0000000000..40a9ff18da
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_stats.c
@@ -0,0 +1,289 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * drivers/cpufreq/cpufreq_stats.c
+ *
+ * Copyright (C) 2003-2004 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
+ * (C) 2004 Zou Nan hai <nanhai.zou@intel.com>.
+ */
+
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+#include <linux/sched/clock.h>
+#include <linux/slab.h>
+
+struct cpufreq_stats {
+ unsigned int total_trans;
+ unsigned long long last_time;
+ unsigned int max_state;
+ unsigned int state_num;
+ unsigned int last_index;
+ u64 *time_in_state;
+ unsigned int *freq_table;
+ unsigned int *trans_table;
+
+ /* Deferred reset */
+ unsigned int reset_pending;
+ unsigned long long reset_time;
+};
+
+static void cpufreq_stats_update(struct cpufreq_stats *stats,
+ unsigned long long time)
+{
+ unsigned long long cur_time = local_clock();
+
+ stats->time_in_state[stats->last_index] += cur_time - time;
+ stats->last_time = cur_time;
+}
+
+static void cpufreq_stats_reset_table(struct cpufreq_stats *stats)
+{
+ unsigned int count = stats->max_state;
+
+ memset(stats->time_in_state, 0, count * sizeof(u64));
+ memset(stats->trans_table, 0, count * count * sizeof(int));
+ stats->last_time = local_clock();
+ stats->total_trans = 0;
+
+ /* Adjust for the time elapsed since reset was requested */
+ WRITE_ONCE(stats->reset_pending, 0);
+ /*
+ * Prevent the reset_time read from being reordered before the
+ * reset_pending accesses in cpufreq_stats_record_transition().
+ */
+ smp_rmb();
+ cpufreq_stats_update(stats, READ_ONCE(stats->reset_time));
+}
+
+static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf)
+{
+ struct cpufreq_stats *stats = policy->stats;
+
+ if (READ_ONCE(stats->reset_pending))
+ return sprintf(buf, "%d\n", 0);
+ else
+ return sprintf(buf, "%u\n", stats->total_trans);
+}
+cpufreq_freq_attr_ro(total_trans);
+
+static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf)
+{
+ struct cpufreq_stats *stats = policy->stats;
+ bool pending = READ_ONCE(stats->reset_pending);
+ unsigned long long time;
+ ssize_t len = 0;
+ int i;
+
+ for (i = 0; i < stats->state_num; i++) {
+ if (pending) {
+ if (i == stats->last_index) {
+ /*
+ * Prevent the reset_time read from occurring
+ * before the reset_pending read above.
+ */
+ smp_rmb();
+ time = local_clock() - READ_ONCE(stats->reset_time);
+ } else {
+ time = 0;
+ }
+ } else {
+ time = stats->time_in_state[i];
+ if (i == stats->last_index)
+ time += local_clock() - stats->last_time;
+ }
+
+ len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i],
+ nsec_to_clock_t(time));
+ }
+ return len;
+}
+cpufreq_freq_attr_ro(time_in_state);
+
+/* We don't care what is written to the attribute */
+static ssize_t store_reset(struct cpufreq_policy *policy, const char *buf,
+ size_t count)
+{
+ struct cpufreq_stats *stats = policy->stats;
+
+ /*
+ * Defer resetting of stats to cpufreq_stats_record_transition() to
+ * avoid races.
+ */
+ WRITE_ONCE(stats->reset_time, local_clock());
+ /*
+ * The memory barrier below is to prevent the readers of reset_time from
+ * seeing a stale or partially updated value.
+ */
+ smp_wmb();
+ WRITE_ONCE(stats->reset_pending, 1);
+
+ return count;
+}
+cpufreq_freq_attr_wo(reset);
+
+static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf)
+{
+ struct cpufreq_stats *stats = policy->stats;
+ bool pending = READ_ONCE(stats->reset_pending);
+ ssize_t len = 0;
+ int i, j, count;
+
+ len += sysfs_emit_at(buf, len, " From : To\n");
+ len += sysfs_emit_at(buf, len, " : ");
+ for (i = 0; i < stats->state_num; i++) {
+ if (len >= PAGE_SIZE - 1)
+ break;
+ len += sysfs_emit_at(buf, len, "%9u ", stats->freq_table[i]);
+ }
+ if (len >= PAGE_SIZE - 1)
+ return PAGE_SIZE - 1;
+
+ len += sysfs_emit_at(buf, len, "\n");
+
+ for (i = 0; i < stats->state_num; i++) {
+ if (len >= PAGE_SIZE - 1)
+ break;
+
+ len += sysfs_emit_at(buf, len, "%9u: ", stats->freq_table[i]);
+
+ for (j = 0; j < stats->state_num; j++) {
+ if (len >= PAGE_SIZE - 1)
+ break;
+
+ if (pending)
+ count = 0;
+ else
+ count = stats->trans_table[i * stats->max_state + j];
+
+ len += sysfs_emit_at(buf, len, "%9u ", count);
+ }
+ if (len >= PAGE_SIZE - 1)
+ break;
+ len += sysfs_emit_at(buf, len, "\n");
+ }
+
+ if (len >= PAGE_SIZE - 1) {
+ pr_warn_once("cpufreq transition table exceeds PAGE_SIZE. Disabling\n");
+ return -EFBIG;
+ }
+ return len;
+}
+cpufreq_freq_attr_ro(trans_table);
+
+static struct attribute *default_attrs[] = {
+ &total_trans.attr,
+ &time_in_state.attr,
+ &reset.attr,
+ &trans_table.attr,
+ NULL
+};
+static const struct attribute_group stats_attr_group = {
+ .attrs = default_attrs,
+ .name = "stats"
+};
+
+static int freq_table_get_index(struct cpufreq_stats *stats, unsigned int freq)
+{
+ int index;
+ for (index = 0; index < stats->max_state; index++)
+ if (stats->freq_table[index] == freq)
+ return index;
+ return -1;
+}
+
+void cpufreq_stats_free_table(struct cpufreq_policy *policy)
+{
+ struct cpufreq_stats *stats = policy->stats;
+
+ /* Already freed */
+ if (!stats)
+ return;
+
+ pr_debug("%s: Free stats table\n", __func__);
+
+ sysfs_remove_group(&policy->kobj, &stats_attr_group);
+ kfree(stats->time_in_state);
+ kfree(stats);
+ policy->stats = NULL;
+}
+
+void cpufreq_stats_create_table(struct cpufreq_policy *policy)
+{
+ unsigned int i = 0, count;
+ struct cpufreq_stats *stats;
+ unsigned int alloc_size;
+ struct cpufreq_frequency_table *pos;
+
+ count = cpufreq_table_count_valid_entries(policy);
+ if (!count)
+ return;
+
+ /* stats already initialized */
+ if (policy->stats)
+ return;
+
+ stats = kzalloc(sizeof(*stats), GFP_KERNEL);
+ if (!stats)
+ return;
+
+ alloc_size = count * sizeof(int) + count * sizeof(u64);
+
+ alloc_size += count * count * sizeof(int);
+
+ /* Allocate memory for time_in_state/freq_table/trans_table in one go */
+ stats->time_in_state = kzalloc(alloc_size, GFP_KERNEL);
+ if (!stats->time_in_state)
+ goto free_stat;
+
+ stats->freq_table = (unsigned int *)(stats->time_in_state + count);
+
+ stats->trans_table = stats->freq_table + count;
+
+ stats->max_state = count;
+
+ /* Find valid-unique entries */
+ cpufreq_for_each_valid_entry(pos, policy->freq_table)
+ if (policy->freq_table_sorted != CPUFREQ_TABLE_UNSORTED ||
+ freq_table_get_index(stats, pos->frequency) == -1)
+ stats->freq_table[i++] = pos->frequency;
+
+ stats->state_num = i;
+ stats->last_time = local_clock();
+ stats->last_index = freq_table_get_index(stats, policy->cur);
+
+ policy->stats = stats;
+ if (!sysfs_create_group(&policy->kobj, &stats_attr_group))
+ return;
+
+ /* We failed, release resources */
+ policy->stats = NULL;
+ kfree(stats->time_in_state);
+free_stat:
+ kfree(stats);
+}
+
+void cpufreq_stats_record_transition(struct cpufreq_policy *policy,
+ unsigned int new_freq)
+{
+ struct cpufreq_stats *stats = policy->stats;
+ int old_index, new_index;
+
+ if (unlikely(!stats))
+ return;
+
+ if (unlikely(READ_ONCE(stats->reset_pending)))
+ cpufreq_stats_reset_table(stats);
+
+ old_index = stats->last_index;
+ new_index = freq_table_get_index(stats, new_freq);
+
+ /* We can't do stats->time_in_state[-1]= .. */
+ if (unlikely(old_index == -1 || new_index == -1 || old_index == new_index))
+ return;
+
+ cpufreq_stats_update(stats, stats->last_time);
+
+ stats->last_index = new_index;
+ stats->trans_table[old_index * stats->max_state + new_index]++;
+ stats->total_trans++;
+}
diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c
new file mode 100644
index 0000000000..50a4d78465
--- /dev/null
+++ b/drivers/cpufreq/cpufreq_userspace.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/*
+ * linux/drivers/cpufreq/cpufreq_userspace.c
+ *
+ * Copyright (C) 2001 Russell King
+ * (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+
+static DEFINE_PER_CPU(unsigned int, cpu_is_managed);
+static DEFINE_MUTEX(userspace_mutex);
+
+/**
+ * cpufreq_set - set the CPU frequency
+ * @policy: pointer to policy struct where freq is being set
+ * @freq: target frequency in kHz
+ *
+ * Sets the CPU frequency to freq.
+ */
+static int cpufreq_set(struct cpufreq_policy *policy, unsigned int freq)
+{
+ int ret = -EINVAL;
+ unsigned int *setspeed = policy->governor_data;
+
+ pr_debug("cpufreq_set for cpu %u, freq %u kHz\n", policy->cpu, freq);
+
+ mutex_lock(&userspace_mutex);
+ if (!per_cpu(cpu_is_managed, policy->cpu))
+ goto err;
+
+ *setspeed = freq;
+
+ ret = __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L);
+ err:
+ mutex_unlock(&userspace_mutex);
+ return ret;
+}
+
+static ssize_t show_speed(struct cpufreq_policy *policy, char *buf)
+{
+ return sprintf(buf, "%u\n", policy->cur);
+}
+
+static int cpufreq_userspace_policy_init(struct cpufreq_policy *policy)
+{
+ unsigned int *setspeed;
+
+ setspeed = kzalloc(sizeof(*setspeed), GFP_KERNEL);
+ if (!setspeed)
+ return -ENOMEM;
+
+ policy->governor_data = setspeed;
+ return 0;
+}
+
+static void cpufreq_userspace_policy_exit(struct cpufreq_policy *policy)
+{
+ mutex_lock(&userspace_mutex);
+ kfree(policy->governor_data);
+ policy->governor_data = NULL;
+ mutex_unlock(&userspace_mutex);
+}
+
+static int cpufreq_userspace_policy_start(struct cpufreq_policy *policy)
+{
+ unsigned int *setspeed = policy->governor_data;
+
+ BUG_ON(!policy->cur);
+ pr_debug("started managing cpu %u\n", policy->cpu);
+
+ mutex_lock(&userspace_mutex);
+ per_cpu(cpu_is_managed, policy->cpu) = 1;
+ *setspeed = policy->cur;
+ mutex_unlock(&userspace_mutex);
+ return 0;
+}
+
+static void cpufreq_userspace_policy_stop(struct cpufreq_policy *policy)
+{
+ unsigned int *setspeed = policy->governor_data;
+
+ pr_debug("managing cpu %u stopped\n", policy->cpu);
+
+ mutex_lock(&userspace_mutex);
+ per_cpu(cpu_is_managed, policy->cpu) = 0;
+ *setspeed = 0;
+ mutex_unlock(&userspace_mutex);
+}
+
+static void cpufreq_userspace_policy_limits(struct cpufreq_policy *policy)
+{
+ unsigned int *setspeed = policy->governor_data;
+
+ mutex_lock(&userspace_mutex);
+
+ pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n",
+ policy->cpu, policy->min, policy->max, policy->cur, *setspeed);
+
+ if (policy->max < *setspeed)
+ __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
+ else if (policy->min > *setspeed)
+ __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
+ else
+ __cpufreq_driver_target(policy, *setspeed, CPUFREQ_RELATION_L);
+
+ mutex_unlock(&userspace_mutex);
+}
+
+static struct cpufreq_governor cpufreq_gov_userspace = {
+ .name = "userspace",
+ .init = cpufreq_userspace_policy_init,
+ .exit = cpufreq_userspace_policy_exit,
+ .start = cpufreq_userspace_policy_start,
+ .stop = cpufreq_userspace_policy_stop,
+ .limits = cpufreq_userspace_policy_limits,
+ .store_setspeed = cpufreq_set,
+ .show_setspeed = show_speed,
+ .owner = THIS_MODULE,
+};
+
+MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>, "
+ "Russell King <rmk@arm.linux.org.uk>");
+MODULE_DESCRIPTION("CPUfreq policy governor 'userspace'");
+MODULE_LICENSE("GPL");
+
+#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE
+struct cpufreq_governor *cpufreq_default_governor(void)
+{
+ return &cpufreq_gov_userspace;
+}
+#endif
+
+cpufreq_governor_init(cpufreq_gov_userspace);
+cpufreq_governor_exit(cpufreq_gov_userspace);
diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c
new file mode 100644
index 0000000000..7d2754411d
--- /dev/null
+++ b/drivers/cpufreq/davinci-cpufreq.c
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * CPU frequency scaling for DaVinci
+ *
+ * Copyright (C) 2009 Texas Instruments Incorporated - https://www.ti.com/
+ *
+ * Based on linux/arch/arm/plat-omap/cpu-omap.c. Original Copyright follows:
+ *
+ * Copyright (C) 2005 Nokia Corporation
+ * Written by Tony Lindgren <tony@atomide.com>
+ *
+ * Based on cpu-sa1110.c, Copyright (C) 2001 Russell King
+ *
+ * Copyright (C) 2007-2008 Texas Instruments, Inc.
+ * Updated to support OMAP3
+ * Rajendra Nayak <rnayak@ti.com>
+ */
+#include <linux/types.h>
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/platform_data/davinci-cpufreq.h>
+#include <linux/platform_device.h>
+#include <linux/export.h>
+
+struct davinci_cpufreq {
+ struct device *dev;
+ struct clk *armclk;
+ struct clk *asyncclk;
+ unsigned long asyncrate;
+};
+static struct davinci_cpufreq cpufreq;
+
+static int davinci_target(struct cpufreq_policy *policy, unsigned int idx)
+{
+ struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data;
+ struct clk *armclk = cpufreq.armclk;
+ unsigned int old_freq, new_freq;
+ int ret = 0;
+
+ old_freq = policy->cur;
+ new_freq = pdata->freq_table[idx].frequency;
+
+ /* if moving to higher frequency, up the voltage beforehand */
+ if (pdata->set_voltage && new_freq > old_freq) {
+ ret = pdata->set_voltage(idx);
+ if (ret)
+ return ret;
+ }
+
+ ret = clk_set_rate(armclk, new_freq * 1000);
+ if (ret)
+ return ret;
+
+ if (cpufreq.asyncclk) {
+ ret = clk_set_rate(cpufreq.asyncclk, cpufreq.asyncrate);
+ if (ret)
+ return ret;
+ }
+
+ /* if moving to lower freq, lower the voltage after lowering freq */
+ if (pdata->set_voltage && new_freq < old_freq)
+ pdata->set_voltage(idx);
+
+ return 0;
+}
+
+static int davinci_cpu_init(struct cpufreq_policy *policy)
+{
+ int result = 0;
+ struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data;
+ struct cpufreq_frequency_table *freq_table = pdata->freq_table;
+
+ if (policy->cpu != 0)
+ return -EINVAL;
+
+ /* Finish platform specific initialization */
+ if (pdata->init) {
+ result = pdata->init();
+ if (result)
+ return result;
+ }
+
+ policy->clk = cpufreq.armclk;
+
+ /*
+ * Time measurement across the target() function yields ~1500-1800us
+ * time taken with no drivers on notification list.
+ * Setting the latency to 2000 us to accommodate addition of drivers
+ * to pre/post change notification list.
+ */
+ cpufreq_generic_init(policy, freq_table, 2000 * 1000);
+ return 0;
+}
+
+static struct cpufreq_driver davinci_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = davinci_target,
+ .get = cpufreq_generic_get,
+ .init = davinci_cpu_init,
+ .name = "davinci",
+ .attr = cpufreq_generic_attr,
+};
+
+static int __init davinci_cpufreq_probe(struct platform_device *pdev)
+{
+ struct davinci_cpufreq_config *pdata = pdev->dev.platform_data;
+ struct clk *asyncclk;
+
+ if (!pdata)
+ return -EINVAL;
+ if (!pdata->freq_table)
+ return -EINVAL;
+
+ cpufreq.dev = &pdev->dev;
+
+ cpufreq.armclk = clk_get(NULL, "arm");
+ if (IS_ERR(cpufreq.armclk)) {
+ dev_err(cpufreq.dev, "Unable to get ARM clock\n");
+ return PTR_ERR(cpufreq.armclk);
+ }
+
+ asyncclk = clk_get(cpufreq.dev, "async");
+ if (!IS_ERR(asyncclk)) {
+ cpufreq.asyncclk = asyncclk;
+ cpufreq.asyncrate = clk_get_rate(asyncclk);
+ }
+
+ return cpufreq_register_driver(&davinci_driver);
+}
+
+static void __exit davinci_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&davinci_driver);
+
+ clk_put(cpufreq.armclk);
+
+ if (cpufreq.asyncclk)
+ clk_put(cpufreq.asyncclk);
+}
+
+static struct platform_driver davinci_cpufreq_driver = {
+ .driver = {
+ .name = "cpufreq-davinci",
+ },
+ .remove_new = __exit_p(davinci_cpufreq_remove),
+};
+
+int __init davinci_cpufreq_init(void)
+{
+ return platform_driver_probe(&davinci_cpufreq_driver,
+ davinci_cpufreq_probe);
+}
+
diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c
new file mode 100644
index 0000000000..ab93bce8ae
--- /dev/null
+++ b/drivers/cpufreq/e_powersaver.c
@@ -0,0 +1,424 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Based on documentation provided by Dave Jones. Thanks!
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/ioport.h>
+#include <linux/slab.h>
+#include <linux/timex.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/msr.h>
+#include <asm/tsc.h>
+
+#if IS_ENABLED(CONFIG_ACPI_PROCESSOR)
+#include <linux/acpi.h>
+#include <acpi/processor.h>
+#endif
+
+#define EPS_BRAND_C7M 0
+#define EPS_BRAND_C7 1
+#define EPS_BRAND_EDEN 2
+#define EPS_BRAND_C3 3
+#define EPS_BRAND_C7D 4
+
+struct eps_cpu_data {
+ u32 fsb;
+#if IS_ENABLED(CONFIG_ACPI_PROCESSOR)
+ u32 bios_limit;
+#endif
+ struct cpufreq_frequency_table freq_table[];
+};
+
+static struct eps_cpu_data *eps_cpu[NR_CPUS];
+
+/* Module parameters */
+static int freq_failsafe_off;
+static int voltage_failsafe_off;
+static int set_max_voltage;
+
+#if IS_ENABLED(CONFIG_ACPI_PROCESSOR)
+static int ignore_acpi_limit;
+
+static struct acpi_processor_performance *eps_acpi_cpu_perf;
+
+/* Minimum necessary to get acpi_processor_get_bios_limit() working */
+static int eps_acpi_init(void)
+{
+ eps_acpi_cpu_perf = kzalloc(sizeof(*eps_acpi_cpu_perf),
+ GFP_KERNEL);
+ if (!eps_acpi_cpu_perf)
+ return -ENOMEM;
+
+ if (!zalloc_cpumask_var(&eps_acpi_cpu_perf->shared_cpu_map,
+ GFP_KERNEL)) {
+ kfree(eps_acpi_cpu_perf);
+ eps_acpi_cpu_perf = NULL;
+ return -ENOMEM;
+ }
+
+ if (acpi_processor_register_performance(eps_acpi_cpu_perf, 0)) {
+ free_cpumask_var(eps_acpi_cpu_perf->shared_cpu_map);
+ kfree(eps_acpi_cpu_perf);
+ eps_acpi_cpu_perf = NULL;
+ return -EIO;
+ }
+ return 0;
+}
+
+static int eps_acpi_exit(struct cpufreq_policy *policy)
+{
+ if (eps_acpi_cpu_perf) {
+ acpi_processor_unregister_performance(0);
+ free_cpumask_var(eps_acpi_cpu_perf->shared_cpu_map);
+ kfree(eps_acpi_cpu_perf);
+ eps_acpi_cpu_perf = NULL;
+ }
+ return 0;
+}
+#endif
+
+static unsigned int eps_get(unsigned int cpu)
+{
+ struct eps_cpu_data *centaur;
+ u32 lo, hi;
+
+ if (cpu)
+ return 0;
+ centaur = eps_cpu[cpu];
+ if (centaur == NULL)
+ return 0;
+
+ /* Return current frequency */
+ rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+ return centaur->fsb * ((lo >> 8) & 0xff);
+}
+
+static int eps_set_state(struct eps_cpu_data *centaur,
+ struct cpufreq_policy *policy,
+ u32 dest_state)
+{
+ u32 lo, hi;
+ int i;
+
+ /* Wait while CPU is busy */
+ rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+ i = 0;
+ while (lo & ((1 << 16) | (1 << 17))) {
+ udelay(16);
+ rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+ i++;
+ if (unlikely(i > 64)) {
+ return -ENODEV;
+ }
+ }
+ /* Set new multiplier and voltage */
+ wrmsr(MSR_IA32_PERF_CTL, dest_state & 0xffff, 0);
+ /* Wait until transition end */
+ i = 0;
+ do {
+ udelay(16);
+ rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+ i++;
+ if (unlikely(i > 64)) {
+ return -ENODEV;
+ }
+ } while (lo & ((1 << 16) | (1 << 17)));
+
+#ifdef DEBUG
+ {
+ u8 current_multiplier, current_voltage;
+
+ /* Print voltage and multiplier */
+ rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+ current_voltage = lo & 0xff;
+ pr_info("Current voltage = %dmV\n", current_voltage * 16 + 700);
+ current_multiplier = (lo >> 8) & 0xff;
+ pr_info("Current multiplier = %d\n", current_multiplier);
+ }
+#endif
+ return 0;
+}
+
+static int eps_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ struct eps_cpu_data *centaur;
+ unsigned int cpu = policy->cpu;
+ unsigned int dest_state;
+ int ret;
+
+ if (unlikely(eps_cpu[cpu] == NULL))
+ return -ENODEV;
+ centaur = eps_cpu[cpu];
+
+ /* Make frequency transition */
+ dest_state = centaur->freq_table[index].driver_data & 0xffff;
+ ret = eps_set_state(centaur, policy, dest_state);
+ if (ret)
+ pr_err("Timeout!\n");
+ return ret;
+}
+
+static int eps_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int i;
+ u32 lo, hi;
+ u64 val;
+ u8 current_multiplier, current_voltage;
+ u8 max_multiplier, max_voltage;
+ u8 min_multiplier, min_voltage;
+ u8 brand = 0;
+ u32 fsb;
+ struct eps_cpu_data *centaur;
+ struct cpuinfo_x86 *c = &cpu_data(0);
+ struct cpufreq_frequency_table *f_table;
+ int k, step, voltage;
+ int states;
+#if IS_ENABLED(CONFIG_ACPI_PROCESSOR)
+ unsigned int limit;
+#endif
+
+ if (policy->cpu != 0)
+ return -ENODEV;
+
+ /* Check brand */
+ pr_info("Detected VIA ");
+
+ switch (c->x86_model) {
+ case 10:
+ rdmsr(0x1153, lo, hi);
+ brand = (((lo >> 2) ^ lo) >> 18) & 3;
+ pr_cont("Model A ");
+ break;
+ case 13:
+ rdmsr(0x1154, lo, hi);
+ brand = (((lo >> 4) ^ (lo >> 2))) & 0x000000ff;
+ pr_cont("Model D ");
+ break;
+ }
+
+ switch (brand) {
+ case EPS_BRAND_C7M:
+ pr_cont("C7-M\n");
+ break;
+ case EPS_BRAND_C7:
+ pr_cont("C7\n");
+ break;
+ case EPS_BRAND_EDEN:
+ pr_cont("Eden\n");
+ break;
+ case EPS_BRAND_C7D:
+ pr_cont("C7-D\n");
+ break;
+ case EPS_BRAND_C3:
+ pr_cont("C3\n");
+ return -ENODEV;
+ }
+ /* Enable Enhanced PowerSaver */
+ rdmsrl(MSR_IA32_MISC_ENABLE, val);
+ if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
+ val |= MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP;
+ wrmsrl(MSR_IA32_MISC_ENABLE, val);
+ /* Can be locked at 0 */
+ rdmsrl(MSR_IA32_MISC_ENABLE, val);
+ if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
+ pr_info("Can't enable Enhanced PowerSaver\n");
+ return -ENODEV;
+ }
+ }
+
+ /* Print voltage and multiplier */
+ rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+ current_voltage = lo & 0xff;
+ pr_info("Current voltage = %dmV\n", current_voltage * 16 + 700);
+ current_multiplier = (lo >> 8) & 0xff;
+ pr_info("Current multiplier = %d\n", current_multiplier);
+
+ /* Print limits */
+ max_voltage = hi & 0xff;
+ pr_info("Highest voltage = %dmV\n", max_voltage * 16 + 700);
+ max_multiplier = (hi >> 8) & 0xff;
+ pr_info("Highest multiplier = %d\n", max_multiplier);
+ min_voltage = (hi >> 16) & 0xff;
+ pr_info("Lowest voltage = %dmV\n", min_voltage * 16 + 700);
+ min_multiplier = (hi >> 24) & 0xff;
+ pr_info("Lowest multiplier = %d\n", min_multiplier);
+
+ /* Sanity checks */
+ if (current_multiplier == 0 || max_multiplier == 0
+ || min_multiplier == 0)
+ return -EINVAL;
+ if (current_multiplier > max_multiplier
+ || max_multiplier <= min_multiplier)
+ return -EINVAL;
+ if (current_voltage > 0x1f || max_voltage > 0x1f)
+ return -EINVAL;
+ if (max_voltage < min_voltage
+ || current_voltage < min_voltage
+ || current_voltage > max_voltage)
+ return -EINVAL;
+
+ /* Check for systems using underclocked CPU */
+ if (!freq_failsafe_off && max_multiplier != current_multiplier) {
+ pr_info("Your processor is running at different frequency then its maximum. Aborting.\n");
+ pr_info("You can use freq_failsafe_off option to disable this check.\n");
+ return -EINVAL;
+ }
+ if (!voltage_failsafe_off && max_voltage != current_voltage) {
+ pr_info("Your processor is running at different voltage then its maximum. Aborting.\n");
+ pr_info("You can use voltage_failsafe_off option to disable this check.\n");
+ return -EINVAL;
+ }
+
+ /* Calc FSB speed */
+ fsb = cpu_khz / current_multiplier;
+
+#if IS_ENABLED(CONFIG_ACPI_PROCESSOR)
+ /* Check for ACPI processor speed limit */
+ if (!ignore_acpi_limit && !eps_acpi_init()) {
+ if (!acpi_processor_get_bios_limit(policy->cpu, &limit)) {
+ pr_info("ACPI limit %u.%uGHz\n",
+ limit/1000000,
+ (limit%1000000)/10000);
+ eps_acpi_exit(policy);
+ /* Check if max_multiplier is in BIOS limits */
+ if (limit && max_multiplier * fsb > limit) {
+ pr_info("Aborting\n");
+ return -EINVAL;
+ }
+ }
+ }
+#endif
+
+ /* Allow user to set lower maximum voltage then that reported
+ * by processor */
+ if (brand == EPS_BRAND_C7M && set_max_voltage) {
+ u32 v;
+
+ /* Change mV to something hardware can use */
+ v = (set_max_voltage - 700) / 16;
+ /* Check if voltage is within limits */
+ if (v >= min_voltage && v <= max_voltage) {
+ pr_info("Setting %dmV as maximum\n", v * 16 + 700);
+ max_voltage = v;
+ }
+ }
+
+ /* Calc number of p-states supported */
+ if (brand == EPS_BRAND_C7M)
+ states = max_multiplier - min_multiplier + 1;
+ else
+ states = 2;
+
+ /* Allocate private data and frequency table for current cpu */
+ centaur = kzalloc(struct_size(centaur, freq_table, states + 1),
+ GFP_KERNEL);
+ if (!centaur)
+ return -ENOMEM;
+ eps_cpu[0] = centaur;
+
+ /* Copy basic values */
+ centaur->fsb = fsb;
+#if IS_ENABLED(CONFIG_ACPI_PROCESSOR)
+ centaur->bios_limit = limit;
+#endif
+
+ /* Fill frequency and MSR value table */
+ f_table = &centaur->freq_table[0];
+ if (brand != EPS_BRAND_C7M) {
+ f_table[0].frequency = fsb * min_multiplier;
+ f_table[0].driver_data = (min_multiplier << 8) | min_voltage;
+ f_table[1].frequency = fsb * max_multiplier;
+ f_table[1].driver_data = (max_multiplier << 8) | max_voltage;
+ f_table[2].frequency = CPUFREQ_TABLE_END;
+ } else {
+ k = 0;
+ step = ((max_voltage - min_voltage) * 256)
+ / (max_multiplier - min_multiplier);
+ for (i = min_multiplier; i <= max_multiplier; i++) {
+ voltage = (k * step) / 256 + min_voltage;
+ f_table[k].frequency = fsb * i;
+ f_table[k].driver_data = (i << 8) | voltage;
+ k++;
+ }
+ f_table[k].frequency = CPUFREQ_TABLE_END;
+ }
+
+ policy->cpuinfo.transition_latency = 140000; /* 844mV -> 700mV in ns */
+ policy->freq_table = &centaur->freq_table[0];
+
+ return 0;
+}
+
+static int eps_cpu_exit(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+
+ /* Bye */
+ kfree(eps_cpu[cpu]);
+ eps_cpu[cpu] = NULL;
+ return 0;
+}
+
+static struct cpufreq_driver eps_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = eps_target,
+ .init = eps_cpu_init,
+ .exit = eps_cpu_exit,
+ .get = eps_get,
+ .name = "e_powersaver",
+ .attr = cpufreq_generic_attr,
+};
+
+
+/* This driver will work only on Centaur C7 processors with
+ * Enhanced SpeedStep/PowerSaver registers */
+static const struct x86_cpu_id eps_cpu_id[] = {
+ X86_MATCH_VENDOR_FAM_FEATURE(CENTAUR, 6, X86_FEATURE_EST, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, eps_cpu_id);
+
+static int __init eps_init(void)
+{
+ if (!x86_match_cpu(eps_cpu_id) || boot_cpu_data.x86_model < 10)
+ return -ENODEV;
+ if (cpufreq_register_driver(&eps_driver))
+ return -EINVAL;
+ return 0;
+}
+
+static void __exit eps_exit(void)
+{
+ cpufreq_unregister_driver(&eps_driver);
+}
+
+/* Allow user to overclock his machine or to change frequency to higher after
+ * unloading module */
+module_param(freq_failsafe_off, int, 0644);
+MODULE_PARM_DESC(freq_failsafe_off, "Disable current vs max frequency check");
+module_param(voltage_failsafe_off, int, 0644);
+MODULE_PARM_DESC(voltage_failsafe_off, "Disable current vs max voltage check");
+#if IS_ENABLED(CONFIG_ACPI_PROCESSOR)
+module_param(ignore_acpi_limit, int, 0644);
+MODULE_PARM_DESC(ignore_acpi_limit, "Don't check ACPI's processor speed limit");
+#endif
+module_param(set_max_voltage, int, 0644);
+MODULE_PARM_DESC(set_max_voltage, "Set maximum CPU voltage (mV) C7-M only");
+
+MODULE_AUTHOR("Rafal Bilski <rafalbilski@interia.pl>");
+MODULE_DESCRIPTION("Enhanced PowerSaver driver for VIA C7 CPU's.");
+MODULE_LICENSE("GPL");
+
+module_init(eps_init);
+module_exit(eps_exit);
diff --git a/drivers/cpufreq/elanfreq.c b/drivers/cpufreq/elanfreq.c
new file mode 100644
index 0000000000..4ce5eb35dc
--- /dev/null
+++ b/drivers/cpufreq/elanfreq.c
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * elanfreq: cpufreq driver for the AMD ELAN family
+ *
+ * (c) Copyright 2002 Robert Schwebel <r.schwebel@pengutronix.de>
+ *
+ * Parts of this code are (c) Sven Geggus <sven@geggus.net>
+ *
+ * All Rights Reserved.
+ *
+ * 2002-02-13: - initial revision for 2.4.18-pre9 by Robert Schwebel
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+
+#include <linux/delay.h>
+#include <linux/cpufreq.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/msr.h>
+#include <linux/timex.h>
+#include <linux/io.h>
+
+#define REG_CSCIR 0x22 /* Chip Setup and Control Index Register */
+#define REG_CSCDR 0x23 /* Chip Setup and Control Data Register */
+
+/* Module parameter */
+static int max_freq;
+
+struct s_elan_multiplier {
+ int clock; /* frequency in kHz */
+ int val40h; /* PMU Force Mode register */
+ int val80h; /* CPU Clock Speed Register */
+};
+
+/*
+ * It is important that the frequencies
+ * are listed in ascending order here!
+ */
+static struct s_elan_multiplier elan_multiplier[] = {
+ {1000, 0x02, 0x18},
+ {2000, 0x02, 0x10},
+ {4000, 0x02, 0x08},
+ {8000, 0x00, 0x00},
+ {16000, 0x00, 0x02},
+ {33000, 0x00, 0x04},
+ {66000, 0x01, 0x04},
+ {99000, 0x01, 0x05}
+};
+
+static struct cpufreq_frequency_table elanfreq_table[] = {
+ {0, 0, 1000},
+ {0, 1, 2000},
+ {0, 2, 4000},
+ {0, 3, 8000},
+ {0, 4, 16000},
+ {0, 5, 33000},
+ {0, 6, 66000},
+ {0, 7, 99000},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+
+/**
+ * elanfreq_get_cpu_frequency: determine current cpu speed
+ *
+ * Finds out at which frequency the CPU of the Elan SOC runs
+ * at the moment. Frequencies from 1 to 33 MHz are generated
+ * the normal way, 66 and 99 MHz are called "Hyperspeed Mode"
+ * and have the rest of the chip running with 33 MHz.
+ */
+
+static unsigned int elanfreq_get_cpu_frequency(unsigned int cpu)
+{
+ u8 clockspeed_reg; /* Clock Speed Register */
+
+ local_irq_disable();
+ outb_p(0x80, REG_CSCIR);
+ clockspeed_reg = inb_p(REG_CSCDR);
+ local_irq_enable();
+
+ if ((clockspeed_reg & 0xE0) == 0xE0)
+ return 0;
+
+ /* Are we in CPU clock multiplied mode (66/99 MHz)? */
+ if ((clockspeed_reg & 0xE0) == 0xC0) {
+ if ((clockspeed_reg & 0x01) == 0)
+ return 66000;
+ else
+ return 99000;
+ }
+
+ /* 33 MHz is not 32 MHz... */
+ if ((clockspeed_reg & 0xE0) == 0xA0)
+ return 33000;
+
+ return (1<<((clockspeed_reg & 0xE0) >> 5)) * 1000;
+}
+
+
+static int elanfreq_target(struct cpufreq_policy *policy,
+ unsigned int state)
+{
+ /*
+ * Access to the Elan's internal registers is indexed via
+ * 0x22: Chip Setup & Control Register Index Register (CSCI)
+ * 0x23: Chip Setup & Control Register Data Register (CSCD)
+ *
+ */
+
+ /*
+ * 0x40 is the Power Management Unit's Force Mode Register.
+ * Bit 6 enables Hyperspeed Mode (66/100 MHz core frequency)
+ */
+
+ local_irq_disable();
+ outb_p(0x40, REG_CSCIR); /* Disable hyperspeed mode */
+ outb_p(0x00, REG_CSCDR);
+ local_irq_enable(); /* wait till internal pipelines and */
+ udelay(1000); /* buffers have cleaned up */
+
+ local_irq_disable();
+
+ /* now, set the CPU clock speed register (0x80) */
+ outb_p(0x80, REG_CSCIR);
+ outb_p(elan_multiplier[state].val80h, REG_CSCDR);
+
+ /* now, the hyperspeed bit in PMU Force Mode Register (0x40) */
+ outb_p(0x40, REG_CSCIR);
+ outb_p(elan_multiplier[state].val40h, REG_CSCDR);
+ udelay(10000);
+ local_irq_enable();
+
+ return 0;
+}
+/*
+ * Module init and exit code
+ */
+
+static int elanfreq_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpuinfo_x86 *c = &cpu_data(0);
+ struct cpufreq_frequency_table *pos;
+
+ /* capability check */
+ if ((c->x86_vendor != X86_VENDOR_AMD) ||
+ (c->x86 != 4) || (c->x86_model != 10))
+ return -ENODEV;
+
+ /* max freq */
+ if (!max_freq)
+ max_freq = elanfreq_get_cpu_frequency(0);
+
+ /* table init */
+ cpufreq_for_each_entry(pos, elanfreq_table)
+ if (pos->frequency > max_freq)
+ pos->frequency = CPUFREQ_ENTRY_INVALID;
+
+ policy->freq_table = elanfreq_table;
+ return 0;
+}
+
+
+#ifndef MODULE
+/**
+ * elanfreq_setup - elanfreq command line parameter parsing
+ *
+ * elanfreq command line parameter. Use:
+ * elanfreq=66000
+ * to set the maximum CPU frequency to 66 MHz. Note that in
+ * case you do not give this boot parameter, the maximum
+ * frequency will fall back to _current_ CPU frequency which
+ * might be lower. If you build this as a module, use the
+ * max_freq module parameter instead.
+ */
+static int __init elanfreq_setup(char *str)
+{
+ max_freq = simple_strtoul(str, &str, 0);
+ pr_warn("You're using the deprecated elanfreq command line option. Use elanfreq.max_freq instead, please!\n");
+ return 1;
+}
+__setup("elanfreq=", elanfreq_setup);
+#endif
+
+
+static struct cpufreq_driver elanfreq_driver = {
+ .get = elanfreq_get_cpu_frequency,
+ .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = elanfreq_target,
+ .init = elanfreq_cpu_init,
+ .name = "elanfreq",
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct x86_cpu_id elan_id[] = {
+ X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 10, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, elan_id);
+
+static int __init elanfreq_init(void)
+{
+ if (!x86_match_cpu(elan_id))
+ return -ENODEV;
+ return cpufreq_register_driver(&elanfreq_driver);
+}
+
+
+static void __exit elanfreq_exit(void)
+{
+ cpufreq_unregister_driver(&elanfreq_driver);
+}
+
+
+module_param(max_freq, int, 0444);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Robert Schwebel <r.schwebel@pengutronix.de>, "
+ "Sven Geggus <sven@geggus.net>");
+MODULE_DESCRIPTION("cpufreq driver for AMD's Elan CPUs");
+
+module_init(elanfreq_init);
+module_exit(elanfreq_exit);
diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c
new file mode 100644
index 0000000000..c4d4643b6c
--- /dev/null
+++ b/drivers/cpufreq/freq_table.c
@@ -0,0 +1,374 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/drivers/cpufreq/freq_table.c
+ *
+ * Copyright (C) 2002 - 2003 Dominik Brodowski
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+
+/*********************************************************************
+ * FREQUENCY TABLE HELPERS *
+ *********************************************************************/
+
+bool policy_has_boost_freq(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *pos, *table = policy->freq_table;
+
+ if (!table)
+ return false;
+
+ cpufreq_for_each_valid_entry(pos, table)
+ if (pos->flags & CPUFREQ_BOOST_FREQ)
+ return true;
+
+ return false;
+}
+EXPORT_SYMBOL_GPL(policy_has_boost_freq);
+
+int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy,
+ struct cpufreq_frequency_table *table)
+{
+ struct cpufreq_frequency_table *pos;
+ unsigned int min_freq = ~0;
+ unsigned int max_freq = 0;
+ unsigned int freq;
+
+ cpufreq_for_each_valid_entry(pos, table) {
+ freq = pos->frequency;
+
+ if (!cpufreq_boost_enabled()
+ && (pos->flags & CPUFREQ_BOOST_FREQ))
+ continue;
+
+ pr_debug("table entry %u: %u kHz\n", (int)(pos - table), freq);
+ if (freq < min_freq)
+ min_freq = freq;
+ if (freq > max_freq)
+ max_freq = freq;
+ }
+
+ policy->min = policy->cpuinfo.min_freq = min_freq;
+ policy->max = max_freq;
+ /*
+ * If the driver has set its own cpuinfo.max_freq above max_freq, leave
+ * it as is.
+ */
+ if (policy->cpuinfo.max_freq < max_freq)
+ policy->max = policy->cpuinfo.max_freq = max_freq;
+
+ if (policy->min == ~0)
+ return -EINVAL;
+ else
+ return 0;
+}
+
+int cpufreq_frequency_table_verify(struct cpufreq_policy_data *policy,
+ struct cpufreq_frequency_table *table)
+{
+ struct cpufreq_frequency_table *pos;
+ unsigned int freq, next_larger = ~0;
+ bool found = false;
+
+ pr_debug("request for verification of policy (%u - %u kHz) for cpu %u\n",
+ policy->min, policy->max, policy->cpu);
+
+ cpufreq_verify_within_cpu_limits(policy);
+
+ cpufreq_for_each_valid_entry(pos, table) {
+ freq = pos->frequency;
+
+ if ((freq >= policy->min) && (freq <= policy->max)) {
+ found = true;
+ break;
+ }
+
+ if ((next_larger > freq) && (freq > policy->max))
+ next_larger = freq;
+ }
+
+ if (!found) {
+ policy->max = next_larger;
+ cpufreq_verify_within_cpu_limits(policy);
+ }
+
+ pr_debug("verification lead to (%u - %u kHz) for cpu %u\n",
+ policy->min, policy->max, policy->cpu);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(cpufreq_frequency_table_verify);
+
+/*
+ * Generic routine to verify policy & frequency table, requires driver to set
+ * policy->freq_table prior to it.
+ */
+int cpufreq_generic_frequency_table_verify(struct cpufreq_policy_data *policy)
+{
+ if (!policy->freq_table)
+ return -ENODEV;
+
+ return cpufreq_frequency_table_verify(policy, policy->freq_table);
+}
+EXPORT_SYMBOL_GPL(cpufreq_generic_frequency_table_verify);
+
+int cpufreq_table_index_unsorted(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ struct cpufreq_frequency_table optimal = {
+ .driver_data = ~0,
+ .frequency = 0,
+ };
+ struct cpufreq_frequency_table suboptimal = {
+ .driver_data = ~0,
+ .frequency = 0,
+ };
+ struct cpufreq_frequency_table *pos;
+ struct cpufreq_frequency_table *table = policy->freq_table;
+ unsigned int freq, diff, i = 0;
+ int index;
+
+ pr_debug("request for target %u kHz (relation: %u) for cpu %u\n",
+ target_freq, relation, policy->cpu);
+
+ switch (relation) {
+ case CPUFREQ_RELATION_H:
+ suboptimal.frequency = ~0;
+ break;
+ case CPUFREQ_RELATION_L:
+ case CPUFREQ_RELATION_C:
+ optimal.frequency = ~0;
+ break;
+ }
+
+ cpufreq_for_each_valid_entry_idx(pos, table, i) {
+ freq = pos->frequency;
+
+ if ((freq < policy->min) || (freq > policy->max))
+ continue;
+ if (freq == target_freq) {
+ optimal.driver_data = i;
+ break;
+ }
+ switch (relation) {
+ case CPUFREQ_RELATION_H:
+ if (freq < target_freq) {
+ if (freq >= optimal.frequency) {
+ optimal.frequency = freq;
+ optimal.driver_data = i;
+ }
+ } else {
+ if (freq <= suboptimal.frequency) {
+ suboptimal.frequency = freq;
+ suboptimal.driver_data = i;
+ }
+ }
+ break;
+ case CPUFREQ_RELATION_L:
+ if (freq > target_freq) {
+ if (freq <= optimal.frequency) {
+ optimal.frequency = freq;
+ optimal.driver_data = i;
+ }
+ } else {
+ if (freq >= suboptimal.frequency) {
+ suboptimal.frequency = freq;
+ suboptimal.driver_data = i;
+ }
+ }
+ break;
+ case CPUFREQ_RELATION_C:
+ diff = abs(freq - target_freq);
+ if (diff < optimal.frequency ||
+ (diff == optimal.frequency &&
+ freq > table[optimal.driver_data].frequency)) {
+ optimal.frequency = diff;
+ optimal.driver_data = i;
+ }
+ break;
+ }
+ }
+ if (optimal.driver_data > i) {
+ if (suboptimal.driver_data > i) {
+ WARN(1, "Invalid frequency table: %d\n", policy->cpu);
+ return 0;
+ }
+
+ index = suboptimal.driver_data;
+ } else
+ index = optimal.driver_data;
+
+ pr_debug("target index is %u, freq is:%u kHz\n", index,
+ table[index].frequency);
+ return index;
+}
+EXPORT_SYMBOL_GPL(cpufreq_table_index_unsorted);
+
+int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy,
+ unsigned int freq)
+{
+ struct cpufreq_frequency_table *pos, *table = policy->freq_table;
+ int idx;
+
+ if (unlikely(!table)) {
+ pr_debug("%s: Unable to find frequency table\n", __func__);
+ return -ENOENT;
+ }
+
+ cpufreq_for_each_valid_entry_idx(pos, table, idx)
+ if (pos->frequency == freq)
+ return idx;
+
+ return -EINVAL;
+}
+EXPORT_SYMBOL_GPL(cpufreq_frequency_table_get_index);
+
+/*
+ * show_available_freqs - show available frequencies for the specified CPU
+ */
+static ssize_t show_available_freqs(struct cpufreq_policy *policy, char *buf,
+ bool show_boost)
+{
+ ssize_t count = 0;
+ struct cpufreq_frequency_table *pos, *table = policy->freq_table;
+
+ if (!table)
+ return -ENODEV;
+
+ cpufreq_for_each_valid_entry(pos, table) {
+ /*
+ * show_boost = true and driver_data = BOOST freq
+ * display BOOST freqs
+ *
+ * show_boost = false and driver_data = BOOST freq
+ * show_boost = true and driver_data != BOOST freq
+ * continue - do not display anything
+ *
+ * show_boost = false and driver_data != BOOST freq
+ * display NON BOOST freqs
+ */
+ if (show_boost ^ (pos->flags & CPUFREQ_BOOST_FREQ))
+ continue;
+
+ count += sprintf(&buf[count], "%d ", pos->frequency);
+ }
+ count += sprintf(&buf[count], "\n");
+
+ return count;
+
+}
+
+#define cpufreq_attr_available_freq(_name) \
+struct freq_attr cpufreq_freq_attr_##_name##_freqs = \
+__ATTR_RO(_name##_frequencies)
+
+/*
+ * scaling_available_frequencies_show - show available normal frequencies for
+ * the specified CPU
+ */
+static ssize_t scaling_available_frequencies_show(struct cpufreq_policy *policy,
+ char *buf)
+{
+ return show_available_freqs(policy, buf, false);
+}
+cpufreq_attr_available_freq(scaling_available);
+EXPORT_SYMBOL_GPL(cpufreq_freq_attr_scaling_available_freqs);
+
+/*
+ * scaling_boost_frequencies_show - show available boost frequencies for
+ * the specified CPU
+ */
+static ssize_t scaling_boost_frequencies_show(struct cpufreq_policy *policy,
+ char *buf)
+{
+ return show_available_freqs(policy, buf, true);
+}
+cpufreq_attr_available_freq(scaling_boost);
+EXPORT_SYMBOL_GPL(cpufreq_freq_attr_scaling_boost_freqs);
+
+struct freq_attr *cpufreq_generic_attr[] = {
+ &cpufreq_freq_attr_scaling_available_freqs,
+ NULL,
+};
+EXPORT_SYMBOL_GPL(cpufreq_generic_attr);
+
+static int set_freq_table_sorted(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *pos, *table = policy->freq_table;
+ struct cpufreq_frequency_table *prev = NULL;
+ int ascending = 0;
+
+ policy->freq_table_sorted = CPUFREQ_TABLE_UNSORTED;
+
+ cpufreq_for_each_valid_entry(pos, table) {
+ if (!prev) {
+ prev = pos;
+ continue;
+ }
+
+ if (pos->frequency == prev->frequency) {
+ pr_warn("Duplicate freq-table entries: %u\n",
+ pos->frequency);
+ return -EINVAL;
+ }
+
+ /* Frequency increased from prev to pos */
+ if (pos->frequency > prev->frequency) {
+ /* But frequency was decreasing earlier */
+ if (ascending < 0) {
+ pr_debug("Freq table is unsorted\n");
+ return 0;
+ }
+
+ ascending++;
+ } else {
+ /* Frequency decreased from prev to pos */
+
+ /* But frequency was increasing earlier */
+ if (ascending > 0) {
+ pr_debug("Freq table is unsorted\n");
+ return 0;
+ }
+
+ ascending--;
+ }
+
+ prev = pos;
+ }
+
+ if (ascending > 0)
+ policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_ASCENDING;
+ else
+ policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_DESCENDING;
+
+ pr_debug("Freq table is sorted in %s order\n",
+ ascending > 0 ? "ascending" : "descending");
+
+ return 0;
+}
+
+int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy)
+{
+ int ret;
+
+ if (!policy->freq_table) {
+ /* Freq table must be passed by drivers with target_index() */
+ if (has_target_index())
+ return -EINVAL;
+
+ return 0;
+ }
+
+ ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
+ if (ret)
+ return ret;
+
+ return set_freq_table_sorted(policy);
+}
+
+MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>");
+MODULE_DESCRIPTION("CPUfreq frequency table helpers");
diff --git a/drivers/cpufreq/gx-suspmod.c b/drivers/cpufreq/gx-suspmod.c
new file mode 100644
index 0000000000..75b3ef7ec6
--- /dev/null
+++ b/drivers/cpufreq/gx-suspmod.c
@@ -0,0 +1,498 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Cyrix MediaGX and NatSemi Geode Suspend Modulation
+ * (C) 2002 Zwane Mwaikambo <zwane@commfireservices.com>
+ * (C) 2002 Hiroshi Miura <miura@da-cha.org>
+ * All Rights Reserved
+ *
+ * The author(s) of this software shall not be held liable for damages
+ * of any nature resulting due to the use of this software. This
+ * software is provided AS-IS with no warranties.
+ *
+ * Theoretical note:
+ *
+ * (see Geode(tm) CS5530 manual (rev.4.1) page.56)
+ *
+ * CPU frequency control on NatSemi Geode GX1/GXLV processor and CS55x0
+ * are based on Suspend Modulation.
+ *
+ * Suspend Modulation works by asserting and de-asserting the SUSP# pin
+ * to CPU(GX1/GXLV) for configurable durations. When asserting SUSP#
+ * the CPU enters an idle state. GX1 stops its core clock when SUSP# is
+ * asserted then power consumption is reduced.
+ *
+ * Suspend Modulation's OFF/ON duration are configurable
+ * with 'Suspend Modulation OFF Count Register'
+ * and 'Suspend Modulation ON Count Register'.
+ * These registers are 8bit counters that represent the number of
+ * 32us intervals which the SUSP# pin is asserted(ON)/de-asserted(OFF)
+ * to the processor.
+ *
+ * These counters define a ratio which is the effective frequency
+ * of operation of the system.
+ *
+ * OFF Count
+ * F_eff = Fgx * ----------------------
+ * OFF Count + ON Count
+ *
+ * 0 <= On Count, Off Count <= 255
+ *
+ * From these limits, we can get register values
+ *
+ * off_duration + on_duration <= MAX_DURATION
+ * on_duration = off_duration * (stock_freq - freq) / freq
+ *
+ * off_duration = (freq * DURATION) / stock_freq
+ * on_duration = DURATION - off_duration
+ *
+ *---------------------------------------------------------------------------
+ *
+ * ChangeLog:
+ * Dec. 12, 2003 Hiroshi Miura <miura@da-cha.org>
+ * - fix on/off register mistake
+ * - fix cpu_khz calc when it stops cpu modulation.
+ *
+ * Dec. 11, 2002 Hiroshi Miura <miura@da-cha.org>
+ * - rewrite for Cyrix MediaGX Cx5510/5520 and
+ * NatSemi Geode Cs5530(A).
+ *
+ * Jul. ??, 2002 Zwane Mwaikambo <zwane@commfireservices.com>
+ * - cs5530_mod patch for 2.4.19-rc1.
+ *
+ *---------------------------------------------------------------------------
+ *
+ * Todo
+ * Test on machines with 5510, 5530, 5530A
+ */
+
+/************************************************************************
+ * Suspend Modulation - Definitions *
+ ************************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/cpufreq.h>
+#include <linux/pci.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/processor-cyrix.h>
+
+/* PCI config registers, all at F0 */
+#define PCI_PMER1 0x80 /* power management enable register 1 */
+#define PCI_PMER2 0x81 /* power management enable register 2 */
+#define PCI_PMER3 0x82 /* power management enable register 3 */
+#define PCI_IRQTC 0x8c /* irq speedup timer counter register:typical 2 to 4ms */
+#define PCI_VIDTC 0x8d /* video speedup timer counter register: typical 50 to 100ms */
+#define PCI_MODOFF 0x94 /* suspend modulation OFF counter register, 1 = 32us */
+#define PCI_MODON 0x95 /* suspend modulation ON counter register */
+#define PCI_SUSCFG 0x96 /* suspend configuration register */
+
+/* PMER1 bits */
+#define GPM (1<<0) /* global power management */
+#define GIT (1<<1) /* globally enable PM device idle timers */
+#define GTR (1<<2) /* globally enable IO traps */
+#define IRQ_SPDUP (1<<3) /* disable clock throttle during interrupt handling */
+#define VID_SPDUP (1<<4) /* disable clock throttle during vga video handling */
+
+/* SUSCFG bits */
+#define SUSMOD (1<<0) /* enable/disable suspend modulation */
+/* the below is supported only with cs5530 (after rev.1.2)/cs5530A */
+#define SMISPDUP (1<<1) /* select how SMI re-enable suspend modulation: */
+ /* IRQTC timer or read SMI speedup disable reg.(F1BAR[08-09h]) */
+#define SUSCFG (1<<2) /* enable powering down a GXLV processor. "Special 3Volt Suspend" mode */
+/* the below is supported only with cs5530A */
+#define PWRSVE_ISA (1<<3) /* stop ISA clock */
+#define PWRSVE (1<<4) /* active idle */
+
+struct gxfreq_params {
+ u8 on_duration;
+ u8 off_duration;
+ u8 pci_suscfg;
+ u8 pci_pmer1;
+ u8 pci_pmer2;
+ struct pci_dev *cs55x0;
+};
+
+static struct gxfreq_params *gx_params;
+static int stock_freq;
+
+/* PCI bus clock - defaults to 30.000 if cpu_khz is not available */
+static int pci_busclk;
+module_param(pci_busclk, int, 0444);
+
+/* maximum duration for which the cpu may be suspended
+ * (32us * MAX_DURATION). If no parameter is given, this defaults
+ * to 255.
+ * Note that this leads to a maximum of 8 ms(!) where the CPU clock
+ * is suspended -- processing power is just 0.39% of what it used to be,
+ * though. 781.25 kHz(!) for a 200 MHz processor -- wow. */
+static int max_duration = 255;
+module_param(max_duration, int, 0444);
+
+/* For the default policy, we want at least some processing power
+ * - let's say 5%. (min = maxfreq / POLICY_MIN_DIV)
+ */
+#define POLICY_MIN_DIV 20
+
+
+/**
+ * we can detect a core multiplier from dir0_lsb
+ * from GX1 datasheet p.56,
+ * MULT[3:0]:
+ * 0000 = SYSCLK multiplied by 4 (test only)
+ * 0001 = SYSCLK multiplied by 10
+ * 0010 = SYSCLK multiplied by 4
+ * 0011 = SYSCLK multiplied by 6
+ * 0100 = SYSCLK multiplied by 9
+ * 0101 = SYSCLK multiplied by 5
+ * 0110 = SYSCLK multiplied by 7
+ * 0111 = SYSCLK multiplied by 8
+ * of 33.3MHz
+ **/
+static int gx_freq_mult[16] = {
+ 4, 10, 4, 6, 9, 5, 7, 8,
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+
+/****************************************************************
+ * Low Level chipset interface *
+ ****************************************************************/
+static struct pci_device_id gx_chipset_tbl[] __initdata = {
+ { PCI_VDEVICE(CYRIX, PCI_DEVICE_ID_CYRIX_5530_LEGACY), },
+ { PCI_VDEVICE(CYRIX, PCI_DEVICE_ID_CYRIX_5520), },
+ { PCI_VDEVICE(CYRIX, PCI_DEVICE_ID_CYRIX_5510), },
+ { 0, },
+};
+MODULE_DEVICE_TABLE(pci, gx_chipset_tbl);
+
+static void gx_write_byte(int reg, int value)
+{
+ pci_write_config_byte(gx_params->cs55x0, reg, value);
+}
+
+/**
+ * gx_detect_chipset:
+ *
+ **/
+static struct pci_dev * __init gx_detect_chipset(void)
+{
+ struct pci_dev *gx_pci = NULL;
+
+ /* detect which companion chip is used */
+ for_each_pci_dev(gx_pci) {
+ if ((pci_match_id(gx_chipset_tbl, gx_pci)) != NULL)
+ return gx_pci;
+ }
+
+ pr_debug("error: no supported chipset found!\n");
+ return NULL;
+}
+
+/**
+ * gx_get_cpuspeed:
+ *
+ * Finds out at which efficient frequency the Cyrix MediaGX/NatSemi
+ * Geode CPU runs.
+ */
+static unsigned int gx_get_cpuspeed(unsigned int cpu)
+{
+ if ((gx_params->pci_suscfg & SUSMOD) == 0)
+ return stock_freq;
+
+ return (stock_freq * gx_params->off_duration)
+ / (gx_params->on_duration + gx_params->off_duration);
+}
+
+/**
+ * gx_validate_speed:
+ * determine current cpu speed
+ *
+ **/
+
+static unsigned int gx_validate_speed(unsigned int khz, u8 *on_duration,
+ u8 *off_duration)
+{
+ unsigned int i;
+ u8 tmp_on, tmp_off;
+ int old_tmp_freq = stock_freq;
+ int tmp_freq;
+
+ *off_duration = 1;
+ *on_duration = 0;
+
+ for (i = max_duration; i > 0; i--) {
+ tmp_off = ((khz * i) / stock_freq) & 0xff;
+ tmp_on = i - tmp_off;
+ tmp_freq = (stock_freq * tmp_off) / i;
+ /* if this relation is closer to khz, use this. If it's equal,
+ * prefer it, too - lower latency */
+ if (abs(tmp_freq - khz) <= abs(old_tmp_freq - khz)) {
+ *on_duration = tmp_on;
+ *off_duration = tmp_off;
+ old_tmp_freq = tmp_freq;
+ }
+ }
+
+ return old_tmp_freq;
+}
+
+
+/**
+ * gx_set_cpuspeed:
+ * set cpu speed in khz.
+ **/
+
+static void gx_set_cpuspeed(struct cpufreq_policy *policy, unsigned int khz)
+{
+ u8 suscfg, pmer1;
+ unsigned int new_khz;
+ unsigned long flags;
+ struct cpufreq_freqs freqs;
+
+ freqs.old = gx_get_cpuspeed(0);
+
+ new_khz = gx_validate_speed(khz, &gx_params->on_duration,
+ &gx_params->off_duration);
+
+ freqs.new = new_khz;
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+ local_irq_save(flags);
+
+ if (new_khz != stock_freq) {
+ /* if new khz == 100% of CPU speed, it is special case */
+ switch (gx_params->cs55x0->device) {
+ case PCI_DEVICE_ID_CYRIX_5530_LEGACY:
+ pmer1 = gx_params->pci_pmer1 | IRQ_SPDUP | VID_SPDUP;
+ /* FIXME: need to test other values -- Zwane,Miura */
+ /* typical 2 to 4ms */
+ gx_write_byte(PCI_IRQTC, 4);
+ /* typical 50 to 100ms */
+ gx_write_byte(PCI_VIDTC, 100);
+ gx_write_byte(PCI_PMER1, pmer1);
+
+ if (gx_params->cs55x0->revision < 0x10) {
+ /* CS5530(rev 1.2, 1.3) */
+ suscfg = gx_params->pci_suscfg|SUSMOD;
+ } else {
+ /* CS5530A,B.. */
+ suscfg = gx_params->pci_suscfg|SUSMOD|PWRSVE;
+ }
+ break;
+ case PCI_DEVICE_ID_CYRIX_5520:
+ case PCI_DEVICE_ID_CYRIX_5510:
+ suscfg = gx_params->pci_suscfg | SUSMOD;
+ break;
+ default:
+ local_irq_restore(flags);
+ pr_debug("fatal: try to set unknown chipset.\n");
+ return;
+ }
+ } else {
+ suscfg = gx_params->pci_suscfg & ~(SUSMOD);
+ gx_params->off_duration = 0;
+ gx_params->on_duration = 0;
+ pr_debug("suspend modulation disabled: cpu runs 100%% speed.\n");
+ }
+
+ gx_write_byte(PCI_MODOFF, gx_params->off_duration);
+ gx_write_byte(PCI_MODON, gx_params->on_duration);
+
+ gx_write_byte(PCI_SUSCFG, suscfg);
+ pci_read_config_byte(gx_params->cs55x0, PCI_SUSCFG, &suscfg);
+
+ local_irq_restore(flags);
+
+ gx_params->pci_suscfg = suscfg;
+
+ cpufreq_freq_transition_end(policy, &freqs, 0);
+
+ pr_debug("suspend modulation w/ duration of ON:%d us, OFF:%d us\n",
+ gx_params->on_duration * 32, gx_params->off_duration * 32);
+ pr_debug("suspend modulation w/ clock speed: %d kHz.\n", freqs.new);
+}
+
+/****************************************************************
+ * High level functions *
+ ****************************************************************/
+
+/*
+ * cpufreq_gx_verify: test if frequency range is valid
+ *
+ * This function checks if a given frequency range in kHz is valid
+ * for the hardware supported by the driver.
+ */
+
+static int cpufreq_gx_verify(struct cpufreq_policy_data *policy)
+{
+ unsigned int tmp_freq = 0;
+ u8 tmp1, tmp2;
+
+ if (!stock_freq || !policy)
+ return -EINVAL;
+
+ policy->cpu = 0;
+ cpufreq_verify_within_limits(policy, (stock_freq / max_duration),
+ stock_freq);
+
+ /* it needs to be assured that at least one supported frequency is
+ * within policy->min and policy->max. If it is not, policy->max
+ * needs to be increased until one frequency is supported.
+ * policy->min may not be decreased, though. This way we guarantee a
+ * specific processing capacity.
+ */
+ tmp_freq = gx_validate_speed(policy->min, &tmp1, &tmp2);
+ if (tmp_freq < policy->min)
+ tmp_freq += stock_freq / max_duration;
+ policy->min = tmp_freq;
+ if (policy->min > policy->max)
+ policy->max = tmp_freq;
+ tmp_freq = gx_validate_speed(policy->max, &tmp1, &tmp2);
+ if (tmp_freq > policy->max)
+ tmp_freq -= stock_freq / max_duration;
+ policy->max = tmp_freq;
+ if (policy->max < policy->min)
+ policy->max = policy->min;
+ cpufreq_verify_within_limits(policy, (stock_freq / max_duration),
+ stock_freq);
+
+ return 0;
+}
+
+/*
+ * cpufreq_gx_target:
+ *
+ */
+static int cpufreq_gx_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ u8 tmp1, tmp2;
+ unsigned int tmp_freq;
+
+ if (!stock_freq || !policy)
+ return -EINVAL;
+
+ policy->cpu = 0;
+
+ tmp_freq = gx_validate_speed(target_freq, &tmp1, &tmp2);
+ while (tmp_freq < policy->min) {
+ tmp_freq += stock_freq / max_duration;
+ tmp_freq = gx_validate_speed(tmp_freq, &tmp1, &tmp2);
+ }
+ while (tmp_freq > policy->max) {
+ tmp_freq -= stock_freq / max_duration;
+ tmp_freq = gx_validate_speed(tmp_freq, &tmp1, &tmp2);
+ }
+
+ gx_set_cpuspeed(policy, tmp_freq);
+
+ return 0;
+}
+
+static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int maxfreq;
+
+ if (!policy || policy->cpu != 0)
+ return -ENODEV;
+
+ /* determine maximum frequency */
+ if (pci_busclk)
+ maxfreq = pci_busclk * gx_freq_mult[getCx86(CX86_DIR1) & 0x0f];
+ else if (cpu_khz)
+ maxfreq = cpu_khz;
+ else
+ maxfreq = 30000 * gx_freq_mult[getCx86(CX86_DIR1) & 0x0f];
+
+ stock_freq = maxfreq;
+
+ pr_debug("cpu max frequency is %d.\n", maxfreq);
+
+ /* setup basic struct for cpufreq API */
+ policy->cpu = 0;
+
+ if (max_duration < POLICY_MIN_DIV)
+ policy->min = maxfreq / max_duration;
+ else
+ policy->min = maxfreq / POLICY_MIN_DIV;
+ policy->max = maxfreq;
+ policy->cpuinfo.min_freq = maxfreq / max_duration;
+ policy->cpuinfo.max_freq = maxfreq;
+
+ return 0;
+}
+
+/*
+ * cpufreq_gx_init:
+ * MediaGX/Geode GX initialize cpufreq driver
+ */
+static struct cpufreq_driver gx_suspmod_driver = {
+ .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
+ .get = gx_get_cpuspeed,
+ .verify = cpufreq_gx_verify,
+ .target = cpufreq_gx_target,
+ .init = cpufreq_gx_cpu_init,
+ .name = "gx-suspmod",
+};
+
+static int __init cpufreq_gx_init(void)
+{
+ int ret;
+ struct gxfreq_params *params;
+ struct pci_dev *gx_pci;
+
+ /* Test if we have the right hardware */
+ gx_pci = gx_detect_chipset();
+ if (gx_pci == NULL)
+ return -ENODEV;
+
+ /* check whether module parameters are sane */
+ if (max_duration > 0xff)
+ max_duration = 0xff;
+
+ pr_debug("geode suspend modulation available.\n");
+
+ params = kzalloc(sizeof(*params), GFP_KERNEL);
+ if (params == NULL)
+ return -ENOMEM;
+
+ params->cs55x0 = gx_pci;
+ gx_params = params;
+
+ /* keep cs55x0 configurations */
+ pci_read_config_byte(params->cs55x0, PCI_SUSCFG, &(params->pci_suscfg));
+ pci_read_config_byte(params->cs55x0, PCI_PMER1, &(params->pci_pmer1));
+ pci_read_config_byte(params->cs55x0, PCI_PMER2, &(params->pci_pmer2));
+ pci_read_config_byte(params->cs55x0, PCI_MODON, &(params->on_duration));
+ pci_read_config_byte(params->cs55x0, PCI_MODOFF,
+ &(params->off_duration));
+
+ ret = cpufreq_register_driver(&gx_suspmod_driver);
+ if (ret) {
+ kfree(params);
+ return ret; /* register error! */
+ }
+
+ return 0;
+}
+
+static void __exit cpufreq_gx_exit(void)
+{
+ cpufreq_unregister_driver(&gx_suspmod_driver);
+ pci_dev_put(gx_params->cs55x0);
+ kfree(gx_params);
+}
+
+MODULE_AUTHOR("Hiroshi Miura <miura@da-cha.org>");
+MODULE_DESCRIPTION("Cpufreq driver for Cyrix MediaGX and NatSemi Geode");
+MODULE_LICENSE("GPL");
+
+module_init(cpufreq_gx_init);
+module_exit(cpufreq_gx_exit);
+
diff --git a/drivers/cpufreq/highbank-cpufreq.c b/drivers/cpufreq/highbank-cpufreq.c
new file mode 100644
index 0000000000..a458647011
--- /dev/null
+++ b/drivers/cpufreq/highbank-cpufreq.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Calxeda, Inc.
+ *
+ * This driver provides the clk notifier callbacks that are used when
+ * the cpufreq-dt driver changes to frequency to alert the highbank
+ * EnergyCore Management Engine (ECME) about the need to change
+ * voltage. The ECME interfaces with the actual voltage regulators.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/of.h>
+#include <linux/pl320-ipc.h>
+#include <linux/platform_device.h>
+
+#define HB_CPUFREQ_CHANGE_NOTE 0x80000001
+#define HB_CPUFREQ_IPC_LEN 7
+#define HB_CPUFREQ_VOLT_RETRIES 15
+
+static int hb_voltage_change(unsigned int freq)
+{
+ u32 msg[HB_CPUFREQ_IPC_LEN] = {HB_CPUFREQ_CHANGE_NOTE, freq / 1000000};
+
+ return pl320_ipc_transmit(msg);
+}
+
+static int hb_cpufreq_clk_notify(struct notifier_block *nb,
+ unsigned long action, void *hclk)
+{
+ struct clk_notifier_data *clk_data = hclk;
+ int i = 0;
+
+ if (action == PRE_RATE_CHANGE) {
+ if (clk_data->new_rate > clk_data->old_rate)
+ while (hb_voltage_change(clk_data->new_rate))
+ if (i++ > HB_CPUFREQ_VOLT_RETRIES)
+ return NOTIFY_BAD;
+ } else if (action == POST_RATE_CHANGE) {
+ if (clk_data->new_rate < clk_data->old_rate)
+ while (hb_voltage_change(clk_data->new_rate))
+ if (i++ > HB_CPUFREQ_VOLT_RETRIES)
+ return NOTIFY_BAD;
+ }
+
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block hb_cpufreq_clk_nb = {
+ .notifier_call = hb_cpufreq_clk_notify,
+};
+
+static int __init hb_cpufreq_driver_init(void)
+{
+ struct platform_device_info devinfo = { .name = "cpufreq-dt", };
+ struct device *cpu_dev;
+ struct clk *cpu_clk;
+ struct device_node *np;
+ int ret;
+
+ if ((!of_machine_is_compatible("calxeda,highbank")) &&
+ (!of_machine_is_compatible("calxeda,ecx-2000")))
+ return -ENODEV;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev) {
+ pr_err("failed to get highbank cpufreq device\n");
+ return -ENODEV;
+ }
+
+ np = of_node_get(cpu_dev->of_node);
+ if (!np) {
+ pr_err("failed to find highbank cpufreq node\n");
+ return -ENOENT;
+ }
+
+ cpu_clk = clk_get(cpu_dev, NULL);
+ if (IS_ERR(cpu_clk)) {
+ ret = PTR_ERR(cpu_clk);
+ pr_err("failed to get cpu0 clock: %d\n", ret);
+ goto out_put_node;
+ }
+
+ ret = clk_notifier_register(cpu_clk, &hb_cpufreq_clk_nb);
+ if (ret) {
+ pr_err("failed to register clk notifier: %d\n", ret);
+ goto out_put_node;
+ }
+
+ /* Instantiate cpufreq-dt */
+ platform_device_register_full(&devinfo);
+
+out_put_node:
+ of_node_put(np);
+ return ret;
+}
+module_init(hb_cpufreq_driver_init);
+
+static const struct of_device_id __maybe_unused hb_cpufreq_of_match[] = {
+ { .compatible = "calxeda,highbank" },
+ { .compatible = "calxeda,ecx-2000" },
+ { },
+};
+MODULE_DEVICE_TABLE(of, hb_cpufreq_of_match);
+
+MODULE_AUTHOR("Mark Langsdorf <mark.langsdorf@calxeda.com>");
+MODULE_DESCRIPTION("Calxeda Highbank cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/ia64-acpi-cpufreq.c b/drivers/cpufreq/ia64-acpi-cpufreq.c
new file mode 100644
index 0000000000..c6bdc45551
--- /dev/null
+++ b/drivers/cpufreq/ia64-acpi-cpufreq.c
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This file provides the ACPI based P-state support. This
+ * module works with generic cpufreq infrastructure. Most of
+ * the code is based on i386 version
+ * (arch/i386/kernel/cpu/cpufreq/acpi-cpufreq.c)
+ *
+ * Copyright (C) 2005 Intel Corp
+ * Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/proc_fs.h>
+#include <asm/io.h>
+#include <linux/uaccess.h>
+#include <asm/pal.h>
+
+#include <linux/acpi.h>
+#include <acpi/processor.h>
+
+MODULE_AUTHOR("Venkatesh Pallipadi");
+MODULE_DESCRIPTION("ACPI Processor P-States Driver");
+MODULE_LICENSE("GPL");
+
+struct cpufreq_acpi_io {
+ struct acpi_processor_performance acpi_data;
+ unsigned int resume;
+};
+
+struct cpufreq_acpi_req {
+ unsigned int cpu;
+ unsigned int state;
+};
+
+static struct cpufreq_acpi_io *acpi_io_data[NR_CPUS];
+
+static struct cpufreq_driver acpi_cpufreq_driver;
+
+
+static int
+processor_set_pstate (
+ u32 value)
+{
+ s64 retval;
+
+ pr_debug("processor_set_pstate\n");
+
+ retval = ia64_pal_set_pstate((u64)value);
+
+ if (retval) {
+ pr_debug("Failed to set freq to 0x%x, with error 0x%llx\n",
+ value, retval);
+ return -ENODEV;
+ }
+ return (int)retval;
+}
+
+
+static int
+processor_get_pstate (
+ u32 *value)
+{
+ u64 pstate_index = 0;
+ s64 retval;
+
+ pr_debug("processor_get_pstate\n");
+
+ retval = ia64_pal_get_pstate(&pstate_index,
+ PAL_GET_PSTATE_TYPE_INSTANT);
+ *value = (u32) pstate_index;
+
+ if (retval)
+ pr_debug("Failed to get current freq with "
+ "error 0x%llx, idx 0x%x\n", retval, *value);
+
+ return (int)retval;
+}
+
+
+/* To be used only after data->acpi_data is initialized */
+static unsigned
+extract_clock (
+ struct cpufreq_acpi_io *data,
+ unsigned value)
+{
+ unsigned long i;
+
+ pr_debug("extract_clock\n");
+
+ for (i = 0; i < data->acpi_data.state_count; i++) {
+ if (value == data->acpi_data.states[i].status)
+ return data->acpi_data.states[i].core_frequency;
+ }
+ return data->acpi_data.states[i-1].core_frequency;
+}
+
+
+static long
+processor_get_freq (
+ void *arg)
+{
+ struct cpufreq_acpi_req *req = arg;
+ unsigned int cpu = req->cpu;
+ struct cpufreq_acpi_io *data = acpi_io_data[cpu];
+ u32 value;
+ int ret;
+
+ pr_debug("processor_get_freq\n");
+ if (smp_processor_id() != cpu)
+ return -EAGAIN;
+
+ /* processor_get_pstate gets the instantaneous frequency */
+ ret = processor_get_pstate(&value);
+ if (ret) {
+ pr_warn("get performance failed with error %d\n", ret);
+ return ret;
+ }
+ return 1000 * extract_clock(data, value);
+}
+
+
+static long
+processor_set_freq (
+ void *arg)
+{
+ struct cpufreq_acpi_req *req = arg;
+ unsigned int cpu = req->cpu;
+ struct cpufreq_acpi_io *data = acpi_io_data[cpu];
+ int ret, state = req->state;
+ u32 value;
+
+ pr_debug("processor_set_freq\n");
+ if (smp_processor_id() != cpu)
+ return -EAGAIN;
+
+ if (state == data->acpi_data.state) {
+ if (unlikely(data->resume)) {
+ pr_debug("Called after resume, resetting to P%d\n", state);
+ data->resume = 0;
+ } else {
+ pr_debug("Already at target state (P%d)\n", state);
+ return 0;
+ }
+ }
+
+ pr_debug("Transitioning from P%d to P%d\n",
+ data->acpi_data.state, state);
+
+ /*
+ * First we write the target state's 'control' value to the
+ * control_register.
+ */
+ value = (u32) data->acpi_data.states[state].control;
+
+ pr_debug("Transitioning to state: 0x%08x\n", value);
+
+ ret = processor_set_pstate(value);
+ if (ret) {
+ pr_warn("Transition failed with error %d\n", ret);
+ return -ENODEV;
+ }
+
+ data->acpi_data.state = state;
+ return 0;
+}
+
+
+static unsigned int
+acpi_cpufreq_get (
+ unsigned int cpu)
+{
+ struct cpufreq_acpi_req req;
+ long ret;
+
+ req.cpu = cpu;
+ ret = work_on_cpu(cpu, processor_get_freq, &req);
+
+ return ret > 0 ? (unsigned int) ret : 0;
+}
+
+
+static int
+acpi_cpufreq_target (
+ struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct cpufreq_acpi_req req;
+
+ req.cpu = policy->cpu;
+ req.state = index;
+
+ return work_on_cpu(req.cpu, processor_set_freq, &req);
+}
+
+static int
+acpi_cpufreq_cpu_init (
+ struct cpufreq_policy *policy)
+{
+ unsigned int i;
+ unsigned int cpu = policy->cpu;
+ struct cpufreq_acpi_io *data;
+ unsigned int result = 0;
+ struct cpufreq_frequency_table *freq_table;
+
+ pr_debug("acpi_cpufreq_cpu_init\n");
+
+ data = kzalloc(sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return (-ENOMEM);
+
+ acpi_io_data[cpu] = data;
+
+ result = acpi_processor_register_performance(&data->acpi_data, cpu);
+
+ if (result)
+ goto err_free;
+
+ /* capability check */
+ if (data->acpi_data.state_count <= 1) {
+ pr_debug("No P-States\n");
+ result = -ENODEV;
+ goto err_unreg;
+ }
+
+ if ((data->acpi_data.control_register.space_id !=
+ ACPI_ADR_SPACE_FIXED_HARDWARE) ||
+ (data->acpi_data.status_register.space_id !=
+ ACPI_ADR_SPACE_FIXED_HARDWARE)) {
+ pr_debug("Unsupported address space [%d, %d]\n",
+ (u32) (data->acpi_data.control_register.space_id),
+ (u32) (data->acpi_data.status_register.space_id));
+ result = -ENODEV;
+ goto err_unreg;
+ }
+
+ /* alloc freq_table */
+ freq_table = kcalloc(data->acpi_data.state_count + 1,
+ sizeof(*freq_table),
+ GFP_KERNEL);
+ if (!freq_table) {
+ result = -ENOMEM;
+ goto err_unreg;
+ }
+
+ /* detect transition latency */
+ policy->cpuinfo.transition_latency = 0;
+ for (i=0; i<data->acpi_data.state_count; i++) {
+ if ((data->acpi_data.states[i].transition_latency * 1000) >
+ policy->cpuinfo.transition_latency) {
+ policy->cpuinfo.transition_latency =
+ data->acpi_data.states[i].transition_latency * 1000;
+ }
+ }
+
+ /* table init */
+ for (i = 0; i <= data->acpi_data.state_count; i++)
+ {
+ if (i < data->acpi_data.state_count) {
+ freq_table[i].frequency =
+ data->acpi_data.states[i].core_frequency * 1000;
+ } else {
+ freq_table[i].frequency = CPUFREQ_TABLE_END;
+ }
+ }
+
+ policy->freq_table = freq_table;
+
+ /* notify BIOS that we exist */
+ acpi_processor_notify_smm(THIS_MODULE);
+
+ pr_info("CPU%u - ACPI performance management activated\n", cpu);
+
+ for (i = 0; i < data->acpi_data.state_count; i++)
+ pr_debug(" %cP%d: %d MHz, %d mW, %d uS, %d uS, 0x%x 0x%x\n",
+ (i == data->acpi_data.state?'*':' '), i,
+ (u32) data->acpi_data.states[i].core_frequency,
+ (u32) data->acpi_data.states[i].power,
+ (u32) data->acpi_data.states[i].transition_latency,
+ (u32) data->acpi_data.states[i].bus_master_latency,
+ (u32) data->acpi_data.states[i].status,
+ (u32) data->acpi_data.states[i].control);
+
+ /* the first call to ->target() should result in us actually
+ * writing something to the appropriate registers. */
+ data->resume = 1;
+
+ return (result);
+
+ err_unreg:
+ acpi_processor_unregister_performance(cpu);
+ err_free:
+ kfree(data);
+ acpi_io_data[cpu] = NULL;
+
+ return (result);
+}
+
+
+static int
+acpi_cpufreq_cpu_exit (
+ struct cpufreq_policy *policy)
+{
+ struct cpufreq_acpi_io *data = acpi_io_data[policy->cpu];
+
+ pr_debug("acpi_cpufreq_cpu_exit\n");
+
+ if (data) {
+ acpi_io_data[policy->cpu] = NULL;
+ acpi_processor_unregister_performance(policy->cpu);
+ kfree(policy->freq_table);
+ kfree(data);
+ }
+
+ return (0);
+}
+
+
+static struct cpufreq_driver acpi_cpufreq_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = acpi_cpufreq_target,
+ .get = acpi_cpufreq_get,
+ .init = acpi_cpufreq_cpu_init,
+ .exit = acpi_cpufreq_cpu_exit,
+ .name = "acpi-cpufreq",
+ .attr = cpufreq_generic_attr,
+};
+
+
+static int __init
+acpi_cpufreq_init (void)
+{
+ pr_debug("acpi_cpufreq_init\n");
+
+ return cpufreq_register_driver(&acpi_cpufreq_driver);
+}
+
+
+static void __exit
+acpi_cpufreq_exit (void)
+{
+ pr_debug("acpi_cpufreq_exit\n");
+
+ cpufreq_unregister_driver(&acpi_cpufreq_driver);
+}
+
+late_initcall(acpi_cpufreq_init);
+module_exit(acpi_cpufreq_exit);
diff --git a/drivers/cpufreq/imx-cpufreq-dt.c b/drivers/cpufreq/imx-cpufreq-dt.c
new file mode 100644
index 0000000000..577bb9e2f1
--- /dev/null
+++ b/drivers/cpufreq/imx-cpufreq-dt.c
@@ -0,0 +1,195 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019 NXP
+ */
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+#include "cpufreq-dt.h"
+
+#define OCOTP_CFG3_SPEED_GRADE_SHIFT 8
+#define OCOTP_CFG3_SPEED_GRADE_MASK (0x3 << 8)
+#define IMX8MN_OCOTP_CFG3_SPEED_GRADE_MASK (0xf << 8)
+#define OCOTP_CFG3_MKT_SEGMENT_SHIFT 6
+#define OCOTP_CFG3_MKT_SEGMENT_MASK (0x3 << 6)
+#define IMX8MP_OCOTP_CFG3_MKT_SEGMENT_SHIFT 5
+#define IMX8MP_OCOTP_CFG3_MKT_SEGMENT_MASK (0x3 << 5)
+
+#define IMX7ULP_MAX_RUN_FREQ 528000
+
+/* cpufreq-dt device registered by imx-cpufreq-dt */
+static struct platform_device *cpufreq_dt_pdev;
+static struct device *cpu_dev;
+static int cpufreq_opp_token;
+
+enum IMX7ULP_CPUFREQ_CLKS {
+ ARM,
+ CORE,
+ SCS_SEL,
+ HSRUN_CORE,
+ HSRUN_SCS_SEL,
+ FIRC,
+};
+
+static struct clk_bulk_data imx7ulp_clks[] = {
+ { .id = "arm" },
+ { .id = "core" },
+ { .id = "scs_sel" },
+ { .id = "hsrun_core" },
+ { .id = "hsrun_scs_sel" },
+ { .id = "firc" },
+};
+
+static unsigned int imx7ulp_get_intermediate(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ return clk_get_rate(imx7ulp_clks[FIRC].clk);
+}
+
+static int imx7ulp_target_intermediate(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ unsigned int newfreq = policy->freq_table[index].frequency;
+
+ clk_set_parent(imx7ulp_clks[SCS_SEL].clk, imx7ulp_clks[FIRC].clk);
+ clk_set_parent(imx7ulp_clks[HSRUN_SCS_SEL].clk, imx7ulp_clks[FIRC].clk);
+
+ if (newfreq > IMX7ULP_MAX_RUN_FREQ)
+ clk_set_parent(imx7ulp_clks[ARM].clk,
+ imx7ulp_clks[HSRUN_CORE].clk);
+ else
+ clk_set_parent(imx7ulp_clks[ARM].clk, imx7ulp_clks[CORE].clk);
+
+ return 0;
+}
+
+static struct cpufreq_dt_platform_data imx7ulp_data = {
+ .target_intermediate = imx7ulp_target_intermediate,
+ .get_intermediate = imx7ulp_get_intermediate,
+};
+
+static int imx_cpufreq_dt_probe(struct platform_device *pdev)
+{
+ struct platform_device *dt_pdev;
+ u32 cell_value, supported_hw[2];
+ int speed_grade, mkt_segment;
+ int ret;
+
+ cpu_dev = get_cpu_device(0);
+
+ if (!of_property_present(cpu_dev->of_node, "cpu-supply"))
+ return -ENODEV;
+
+ if (of_machine_is_compatible("fsl,imx7ulp")) {
+ ret = clk_bulk_get(cpu_dev, ARRAY_SIZE(imx7ulp_clks),
+ imx7ulp_clks);
+ if (ret)
+ return ret;
+
+ dt_pdev = platform_device_register_data(NULL, "cpufreq-dt",
+ -1, &imx7ulp_data,
+ sizeof(imx7ulp_data));
+ if (IS_ERR(dt_pdev)) {
+ clk_bulk_put(ARRAY_SIZE(imx7ulp_clks), imx7ulp_clks);
+ ret = PTR_ERR(dt_pdev);
+ dev_err(&pdev->dev, "Failed to register cpufreq-dt: %d\n", ret);
+ return ret;
+ }
+
+ cpufreq_dt_pdev = dt_pdev;
+
+ return 0;
+ }
+
+ ret = nvmem_cell_read_u32(cpu_dev, "speed_grade", &cell_value);
+ if (ret)
+ return ret;
+
+ if (of_machine_is_compatible("fsl,imx8mn") ||
+ of_machine_is_compatible("fsl,imx8mp"))
+ speed_grade = (cell_value & IMX8MN_OCOTP_CFG3_SPEED_GRADE_MASK)
+ >> OCOTP_CFG3_SPEED_GRADE_SHIFT;
+ else
+ speed_grade = (cell_value & OCOTP_CFG3_SPEED_GRADE_MASK)
+ >> OCOTP_CFG3_SPEED_GRADE_SHIFT;
+
+ if (of_machine_is_compatible("fsl,imx8mp"))
+ mkt_segment = (cell_value & IMX8MP_OCOTP_CFG3_MKT_SEGMENT_MASK)
+ >> IMX8MP_OCOTP_CFG3_MKT_SEGMENT_SHIFT;
+ else
+ mkt_segment = (cell_value & OCOTP_CFG3_MKT_SEGMENT_MASK)
+ >> OCOTP_CFG3_MKT_SEGMENT_SHIFT;
+
+ /*
+ * Early samples without fuses written report "0 0" which may NOT
+ * match any OPP defined in DT. So clamp to minimum OPP defined in
+ * DT to avoid warning for "no OPPs".
+ *
+ * Applies to i.MX8M series SoCs.
+ */
+ if (mkt_segment == 0 && speed_grade == 0) {
+ if (of_machine_is_compatible("fsl,imx8mm") ||
+ of_machine_is_compatible("fsl,imx8mq"))
+ speed_grade = 1;
+ if (of_machine_is_compatible("fsl,imx8mn") ||
+ of_machine_is_compatible("fsl,imx8mp"))
+ speed_grade = 0xb;
+ }
+
+ supported_hw[0] = BIT(speed_grade);
+ supported_hw[1] = BIT(mkt_segment);
+ dev_info(&pdev->dev, "cpu speed grade %d mkt segment %d supported-hw %#x %#x\n",
+ speed_grade, mkt_segment, supported_hw[0], supported_hw[1]);
+
+ cpufreq_opp_token = dev_pm_opp_set_supported_hw(cpu_dev, supported_hw, 2);
+ if (cpufreq_opp_token < 0) {
+ ret = cpufreq_opp_token;
+ dev_err(&pdev->dev, "Failed to set supported opp: %d\n", ret);
+ return ret;
+ }
+
+ cpufreq_dt_pdev = platform_device_register_data(
+ &pdev->dev, "cpufreq-dt", -1, NULL, 0);
+ if (IS_ERR(cpufreq_dt_pdev)) {
+ dev_pm_opp_put_supported_hw(cpufreq_opp_token);
+ ret = PTR_ERR(cpufreq_dt_pdev);
+ dev_err(&pdev->dev, "Failed to register cpufreq-dt: %d\n", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void imx_cpufreq_dt_remove(struct platform_device *pdev)
+{
+ platform_device_unregister(cpufreq_dt_pdev);
+ if (!of_machine_is_compatible("fsl,imx7ulp"))
+ dev_pm_opp_put_supported_hw(cpufreq_opp_token);
+ else
+ clk_bulk_put(ARRAY_SIZE(imx7ulp_clks), imx7ulp_clks);
+}
+
+static struct platform_driver imx_cpufreq_dt_driver = {
+ .probe = imx_cpufreq_dt_probe,
+ .remove_new = imx_cpufreq_dt_remove,
+ .driver = {
+ .name = "imx-cpufreq-dt",
+ },
+};
+module_platform_driver(imx_cpufreq_dt_driver);
+
+MODULE_ALIAS("platform:imx-cpufreq-dt");
+MODULE_DESCRIPTION("Freescale i.MX cpufreq speed grading driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c
new file mode 100644
index 0000000000..33728c242f
--- /dev/null
+++ b/drivers/cpufreq/imx6q-cpufreq.c
@@ -0,0 +1,547 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2013 Freescale Semiconductor, Inc.
+ */
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/pm_opp.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+
+#define PU_SOC_VOLTAGE_NORMAL 1250000
+#define PU_SOC_VOLTAGE_HIGH 1275000
+#define FREQ_1P2_GHZ 1200000000
+
+static struct regulator *arm_reg;
+static struct regulator *pu_reg;
+static struct regulator *soc_reg;
+
+enum IMX6_CPUFREQ_CLKS {
+ ARM,
+ PLL1_SYS,
+ STEP,
+ PLL1_SW,
+ PLL2_PFD2_396M,
+ /* MX6UL requires two more clks */
+ PLL2_BUS,
+ SECONDARY_SEL,
+};
+#define IMX6Q_CPUFREQ_CLK_NUM 5
+#define IMX6UL_CPUFREQ_CLK_NUM 7
+
+static int num_clks;
+static struct clk_bulk_data clks[] = {
+ { .id = "arm" },
+ { .id = "pll1_sys" },
+ { .id = "step" },
+ { .id = "pll1_sw" },
+ { .id = "pll2_pfd2_396m" },
+ { .id = "pll2_bus" },
+ { .id = "secondary_sel" },
+};
+
+static struct device *cpu_dev;
+static struct cpufreq_frequency_table *freq_table;
+static unsigned int max_freq;
+static unsigned int transition_latency;
+
+static u32 *imx6_soc_volt;
+static u32 soc_opp_count;
+
+static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ struct dev_pm_opp *opp;
+ unsigned long freq_hz, volt, volt_old;
+ unsigned int old_freq, new_freq;
+ bool pll1_sys_temp_enabled = false;
+ int ret;
+
+ new_freq = freq_table[index].frequency;
+ freq_hz = new_freq * 1000;
+ old_freq = clk_get_rate(clks[ARM].clk) / 1000;
+
+ opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_hz);
+ if (IS_ERR(opp)) {
+ dev_err(cpu_dev, "failed to find OPP for %ld\n", freq_hz);
+ return PTR_ERR(opp);
+ }
+
+ volt = dev_pm_opp_get_voltage(opp);
+ dev_pm_opp_put(opp);
+
+ volt_old = regulator_get_voltage(arm_reg);
+
+ dev_dbg(cpu_dev, "%u MHz, %ld mV --> %u MHz, %ld mV\n",
+ old_freq / 1000, volt_old / 1000,
+ new_freq / 1000, volt / 1000);
+
+ /* scaling up? scale voltage before frequency */
+ if (new_freq > old_freq) {
+ if (!IS_ERR(pu_reg)) {
+ ret = regulator_set_voltage_tol(pu_reg, imx6_soc_volt[index], 0);
+ if (ret) {
+ dev_err(cpu_dev, "failed to scale vddpu up: %d\n", ret);
+ return ret;
+ }
+ }
+ ret = regulator_set_voltage_tol(soc_reg, imx6_soc_volt[index], 0);
+ if (ret) {
+ dev_err(cpu_dev, "failed to scale vddsoc up: %d\n", ret);
+ return ret;
+ }
+ ret = regulator_set_voltage_tol(arm_reg, volt, 0);
+ if (ret) {
+ dev_err(cpu_dev,
+ "failed to scale vddarm up: %d\n", ret);
+ return ret;
+ }
+ }
+
+ /*
+ * The setpoints are selected per PLL/PDF frequencies, so we need to
+ * reprogram PLL for frequency scaling. The procedure of reprogramming
+ * PLL1 is as below.
+ * For i.MX6UL, it has a secondary clk mux, the cpu frequency change
+ * flow is slightly different from other i.MX6 OSC.
+ * The cpu frequeny change flow for i.MX6(except i.MX6UL) is as below:
+ * - Enable pll2_pfd2_396m_clk and reparent pll1_sw_clk to it
+ * - Reprogram pll1_sys_clk and reparent pll1_sw_clk back to it
+ * - Disable pll2_pfd2_396m_clk
+ */
+ if (of_machine_is_compatible("fsl,imx6ul") ||
+ of_machine_is_compatible("fsl,imx6ull")) {
+ /*
+ * When changing pll1_sw_clk's parent to pll1_sys_clk,
+ * CPU may run at higher than 528MHz, this will lead to
+ * the system unstable if the voltage is lower than the
+ * voltage of 528MHz, so lower the CPU frequency to one
+ * half before changing CPU frequency.
+ */
+ clk_set_rate(clks[ARM].clk, (old_freq >> 1) * 1000);
+ clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk);
+ if (freq_hz > clk_get_rate(clks[PLL2_PFD2_396M].clk))
+ clk_set_parent(clks[SECONDARY_SEL].clk,
+ clks[PLL2_BUS].clk);
+ else
+ clk_set_parent(clks[SECONDARY_SEL].clk,
+ clks[PLL2_PFD2_396M].clk);
+ clk_set_parent(clks[STEP].clk, clks[SECONDARY_SEL].clk);
+ clk_set_parent(clks[PLL1_SW].clk, clks[STEP].clk);
+ if (freq_hz > clk_get_rate(clks[PLL2_BUS].clk)) {
+ clk_set_rate(clks[PLL1_SYS].clk, new_freq * 1000);
+ clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk);
+ }
+ } else {
+ clk_set_parent(clks[STEP].clk, clks[PLL2_PFD2_396M].clk);
+ clk_set_parent(clks[PLL1_SW].clk, clks[STEP].clk);
+ if (freq_hz > clk_get_rate(clks[PLL2_PFD2_396M].clk)) {
+ clk_set_rate(clks[PLL1_SYS].clk, new_freq * 1000);
+ clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk);
+ } else {
+ /* pll1_sys needs to be enabled for divider rate change to work. */
+ pll1_sys_temp_enabled = true;
+ clk_prepare_enable(clks[PLL1_SYS].clk);
+ }
+ }
+
+ /* Ensure the arm clock divider is what we expect */
+ ret = clk_set_rate(clks[ARM].clk, new_freq * 1000);
+ if (ret) {
+ int ret1;
+
+ dev_err(cpu_dev, "failed to set clock rate: %d\n", ret);
+ ret1 = regulator_set_voltage_tol(arm_reg, volt_old, 0);
+ if (ret1)
+ dev_warn(cpu_dev,
+ "failed to restore vddarm voltage: %d\n", ret1);
+ return ret;
+ }
+
+ /* PLL1 is only needed until after ARM-PODF is set. */
+ if (pll1_sys_temp_enabled)
+ clk_disable_unprepare(clks[PLL1_SYS].clk);
+
+ /* scaling down? scale voltage after frequency */
+ if (new_freq < old_freq) {
+ ret = regulator_set_voltage_tol(arm_reg, volt, 0);
+ if (ret)
+ dev_warn(cpu_dev,
+ "failed to scale vddarm down: %d\n", ret);
+ ret = regulator_set_voltage_tol(soc_reg, imx6_soc_volt[index], 0);
+ if (ret)
+ dev_warn(cpu_dev, "failed to scale vddsoc down: %d\n", ret);
+ if (!IS_ERR(pu_reg)) {
+ ret = regulator_set_voltage_tol(pu_reg, imx6_soc_volt[index], 0);
+ if (ret)
+ dev_warn(cpu_dev, "failed to scale vddpu down: %d\n", ret);
+ }
+ }
+
+ return 0;
+}
+
+static int imx6q_cpufreq_init(struct cpufreq_policy *policy)
+{
+ policy->clk = clks[ARM].clk;
+ cpufreq_generic_init(policy, freq_table, transition_latency);
+ policy->suspend_freq = max_freq;
+
+ return 0;
+}
+
+static struct cpufreq_driver imx6q_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = imx6q_set_target,
+ .get = cpufreq_generic_get,
+ .init = imx6q_cpufreq_init,
+ .register_em = cpufreq_register_em_with_opp,
+ .name = "imx6q-cpufreq",
+ .attr = cpufreq_generic_attr,
+ .suspend = cpufreq_generic_suspend,
+};
+
+static void imx6x_disable_freq_in_opp(struct device *dev, unsigned long freq)
+{
+ int ret = dev_pm_opp_disable(dev, freq);
+
+ if (ret < 0 && ret != -ENODEV)
+ dev_warn(dev, "failed to disable %ldMHz OPP\n", freq / 1000000);
+}
+
+#define OCOTP_CFG3 0x440
+#define OCOTP_CFG3_SPEED_SHIFT 16
+#define OCOTP_CFG3_SPEED_1P2GHZ 0x3
+#define OCOTP_CFG3_SPEED_996MHZ 0x2
+#define OCOTP_CFG3_SPEED_852MHZ 0x1
+
+static int imx6q_opp_check_speed_grading(struct device *dev)
+{
+ struct device_node *np;
+ void __iomem *base;
+ u32 val;
+ int ret;
+
+ if (of_property_present(dev->of_node, "nvmem-cells")) {
+ ret = nvmem_cell_read_u32(dev, "speed_grade", &val);
+ if (ret)
+ return ret;
+ } else {
+ np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-ocotp");
+ if (!np)
+ return -ENOENT;
+
+ base = of_iomap(np, 0);
+ of_node_put(np);
+ if (!base) {
+ dev_err(dev, "failed to map ocotp\n");
+ return -EFAULT;
+ }
+
+ /*
+ * SPEED_GRADING[1:0] defines the max speed of ARM:
+ * 2b'11: 1200000000Hz;
+ * 2b'10: 996000000Hz;
+ * 2b'01: 852000000Hz; -- i.MX6Q Only, exclusive with 996MHz.
+ * 2b'00: 792000000Hz;
+ * We need to set the max speed of ARM according to fuse map.
+ */
+ val = readl_relaxed(base + OCOTP_CFG3);
+ iounmap(base);
+ }
+
+ val >>= OCOTP_CFG3_SPEED_SHIFT;
+ val &= 0x3;
+
+ if (val < OCOTP_CFG3_SPEED_996MHZ)
+ imx6x_disable_freq_in_opp(dev, 996000000);
+
+ if (of_machine_is_compatible("fsl,imx6q") ||
+ of_machine_is_compatible("fsl,imx6qp")) {
+ if (val != OCOTP_CFG3_SPEED_852MHZ)
+ imx6x_disable_freq_in_opp(dev, 852000000);
+
+ if (val != OCOTP_CFG3_SPEED_1P2GHZ)
+ imx6x_disable_freq_in_opp(dev, 1200000000);
+ }
+
+ return 0;
+}
+
+#define OCOTP_CFG3_6UL_SPEED_696MHZ 0x2
+#define OCOTP_CFG3_6ULL_SPEED_792MHZ 0x2
+#define OCOTP_CFG3_6ULL_SPEED_900MHZ 0x3
+
+static int imx6ul_opp_check_speed_grading(struct device *dev)
+{
+ u32 val;
+ int ret = 0;
+
+ if (of_property_present(dev->of_node, "nvmem-cells")) {
+ ret = nvmem_cell_read_u32(dev, "speed_grade", &val);
+ if (ret)
+ return ret;
+ } else {
+ struct device_node *np;
+ void __iomem *base;
+
+ np = of_find_compatible_node(NULL, NULL, "fsl,imx6ul-ocotp");
+ if (!np)
+ np = of_find_compatible_node(NULL, NULL,
+ "fsl,imx6ull-ocotp");
+ if (!np)
+ return -ENOENT;
+
+ base = of_iomap(np, 0);
+ of_node_put(np);
+ if (!base) {
+ dev_err(dev, "failed to map ocotp\n");
+ return -EFAULT;
+ }
+
+ val = readl_relaxed(base + OCOTP_CFG3);
+ iounmap(base);
+ }
+
+ /*
+ * Speed GRADING[1:0] defines the max speed of ARM:
+ * 2b'00: Reserved;
+ * 2b'01: 528000000Hz;
+ * 2b'10: 696000000Hz on i.MX6UL, 792000000Hz on i.MX6ULL;
+ * 2b'11: 900000000Hz on i.MX6ULL only;
+ * We need to set the max speed of ARM according to fuse map.
+ */
+ val >>= OCOTP_CFG3_SPEED_SHIFT;
+ val &= 0x3;
+
+ if (of_machine_is_compatible("fsl,imx6ul"))
+ if (val != OCOTP_CFG3_6UL_SPEED_696MHZ)
+ imx6x_disable_freq_in_opp(dev, 696000000);
+
+ if (of_machine_is_compatible("fsl,imx6ull")) {
+ if (val < OCOTP_CFG3_6ULL_SPEED_792MHZ)
+ imx6x_disable_freq_in_opp(dev, 792000000);
+
+ if (val != OCOTP_CFG3_6ULL_SPEED_900MHZ)
+ imx6x_disable_freq_in_opp(dev, 900000000);
+ }
+
+ return ret;
+}
+
+static int imx6q_cpufreq_probe(struct platform_device *pdev)
+{
+ struct device_node *np;
+ struct dev_pm_opp *opp;
+ unsigned long min_volt, max_volt;
+ int num, ret;
+ const struct property *prop;
+ const __be32 *val;
+ u32 nr, i, j;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev) {
+ pr_err("failed to get cpu0 device\n");
+ return -ENODEV;
+ }
+
+ np = of_node_get(cpu_dev->of_node);
+ if (!np) {
+ dev_err(cpu_dev, "failed to find cpu0 node\n");
+ return -ENOENT;
+ }
+
+ if (of_machine_is_compatible("fsl,imx6ul") ||
+ of_machine_is_compatible("fsl,imx6ull"))
+ num_clks = IMX6UL_CPUFREQ_CLK_NUM;
+ else
+ num_clks = IMX6Q_CPUFREQ_CLK_NUM;
+
+ ret = clk_bulk_get(cpu_dev, num_clks, clks);
+ if (ret)
+ goto put_node;
+
+ arm_reg = regulator_get(cpu_dev, "arm");
+ pu_reg = regulator_get_optional(cpu_dev, "pu");
+ soc_reg = regulator_get(cpu_dev, "soc");
+ if (PTR_ERR(arm_reg) == -EPROBE_DEFER ||
+ PTR_ERR(soc_reg) == -EPROBE_DEFER ||
+ PTR_ERR(pu_reg) == -EPROBE_DEFER) {
+ ret = -EPROBE_DEFER;
+ dev_dbg(cpu_dev, "regulators not ready, defer\n");
+ goto put_reg;
+ }
+ if (IS_ERR(arm_reg) || IS_ERR(soc_reg)) {
+ dev_err(cpu_dev, "failed to get regulators\n");
+ ret = -ENOENT;
+ goto put_reg;
+ }
+
+ ret = dev_pm_opp_of_add_table(cpu_dev);
+ if (ret < 0) {
+ dev_err(cpu_dev, "failed to init OPP table: %d\n", ret);
+ goto put_reg;
+ }
+
+ if (of_machine_is_compatible("fsl,imx6ul") ||
+ of_machine_is_compatible("fsl,imx6ull")) {
+ ret = imx6ul_opp_check_speed_grading(cpu_dev);
+ } else {
+ ret = imx6q_opp_check_speed_grading(cpu_dev);
+ }
+ if (ret) {
+ dev_err_probe(cpu_dev, ret, "failed to read ocotp\n");
+ goto out_free_opp;
+ }
+
+ num = dev_pm_opp_get_opp_count(cpu_dev);
+ if (num < 0) {
+ ret = num;
+ dev_err(cpu_dev, "no OPP table is found: %d\n", ret);
+ goto out_free_opp;
+ }
+
+ ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
+ if (ret) {
+ dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
+ goto out_free_opp;
+ }
+
+ /* Make imx6_soc_volt array's size same as arm opp number */
+ imx6_soc_volt = devm_kcalloc(cpu_dev, num, sizeof(*imx6_soc_volt),
+ GFP_KERNEL);
+ if (imx6_soc_volt == NULL) {
+ ret = -ENOMEM;
+ goto free_freq_table;
+ }
+
+ prop = of_find_property(np, "fsl,soc-operating-points", NULL);
+ if (!prop || !prop->value)
+ goto soc_opp_out;
+
+ /*
+ * Each OPP is a set of tuples consisting of frequency and
+ * voltage like <freq-kHz vol-uV>.
+ */
+ nr = prop->length / sizeof(u32);
+ if (nr % 2 || (nr / 2) < num)
+ goto soc_opp_out;
+
+ for (j = 0; j < num; j++) {
+ val = prop->value;
+ for (i = 0; i < nr / 2; i++) {
+ unsigned long freq = be32_to_cpup(val++);
+ unsigned long volt = be32_to_cpup(val++);
+ if (freq_table[j].frequency == freq) {
+ imx6_soc_volt[soc_opp_count++] = volt;
+ break;
+ }
+ }
+ }
+
+soc_opp_out:
+ /* use fixed soc opp volt if no valid soc opp info found in dtb */
+ if (soc_opp_count != num) {
+ dev_warn(cpu_dev, "can NOT find valid fsl,soc-operating-points property in dtb, use default value!\n");
+ for (j = 0; j < num; j++)
+ imx6_soc_volt[j] = PU_SOC_VOLTAGE_NORMAL;
+ if (freq_table[num - 1].frequency * 1000 == FREQ_1P2_GHZ)
+ imx6_soc_volt[num - 1] = PU_SOC_VOLTAGE_HIGH;
+ }
+
+ if (of_property_read_u32(np, "clock-latency", &transition_latency))
+ transition_latency = CPUFREQ_ETERNAL;
+
+ /*
+ * Calculate the ramp time for max voltage change in the
+ * VDDSOC and VDDPU regulators.
+ */
+ ret = regulator_set_voltage_time(soc_reg, imx6_soc_volt[0], imx6_soc_volt[num - 1]);
+ if (ret > 0)
+ transition_latency += ret * 1000;
+ if (!IS_ERR(pu_reg)) {
+ ret = regulator_set_voltage_time(pu_reg, imx6_soc_volt[0], imx6_soc_volt[num - 1]);
+ if (ret > 0)
+ transition_latency += ret * 1000;
+ }
+
+ /*
+ * OPP is maintained in order of increasing frequency, and
+ * freq_table initialised from OPP is therefore sorted in the
+ * same order.
+ */
+ max_freq = freq_table[--num].frequency;
+ opp = dev_pm_opp_find_freq_exact(cpu_dev,
+ freq_table[0].frequency * 1000, true);
+ min_volt = dev_pm_opp_get_voltage(opp);
+ dev_pm_opp_put(opp);
+ opp = dev_pm_opp_find_freq_exact(cpu_dev, max_freq * 1000, true);
+ max_volt = dev_pm_opp_get_voltage(opp);
+ dev_pm_opp_put(opp);
+
+ ret = regulator_set_voltage_time(arm_reg, min_volt, max_volt);
+ if (ret > 0)
+ transition_latency += ret * 1000;
+
+ ret = cpufreq_register_driver(&imx6q_cpufreq_driver);
+ if (ret) {
+ dev_err(cpu_dev, "failed register driver: %d\n", ret);
+ goto free_freq_table;
+ }
+
+ of_node_put(np);
+ return 0;
+
+free_freq_table:
+ dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
+out_free_opp:
+ dev_pm_opp_of_remove_table(cpu_dev);
+put_reg:
+ if (!IS_ERR(arm_reg))
+ regulator_put(arm_reg);
+ if (!IS_ERR(pu_reg))
+ regulator_put(pu_reg);
+ if (!IS_ERR(soc_reg))
+ regulator_put(soc_reg);
+
+ clk_bulk_put(num_clks, clks);
+put_node:
+ of_node_put(np);
+
+ return ret;
+}
+
+static void imx6q_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&imx6q_cpufreq_driver);
+ dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
+ dev_pm_opp_of_remove_table(cpu_dev);
+ regulator_put(arm_reg);
+ if (!IS_ERR(pu_reg))
+ regulator_put(pu_reg);
+ regulator_put(soc_reg);
+
+ clk_bulk_put(num_clks, clks);
+}
+
+static struct platform_driver imx6q_cpufreq_platdrv = {
+ .driver = {
+ .name = "imx6q-cpufreq",
+ },
+ .probe = imx6q_cpufreq_probe,
+ .remove_new = imx6q_cpufreq_remove,
+};
+module_platform_driver(imx6q_cpufreq_platdrv);
+
+MODULE_ALIAS("platform:imx6q-cpufreq");
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_DESCRIPTION("Freescale i.MX6Q cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
new file mode 100644
index 0000000000..c352a593e5
--- /dev/null
+++ b/drivers/cpufreq/intel_pstate.c
@@ -0,0 +1,3571 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * intel_pstate.c: Native P state management for Intel processors
+ *
+ * (C) Copyright 2012 Intel Corporation
+ * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/kernel_stat.h>
+#include <linux/module.h>
+#include <linux/ktime.h>
+#include <linux/hrtimer.h>
+#include <linux/tick.h>
+#include <linux/slab.h>
+#include <linux/sched/cpufreq.h>
+#include <linux/list.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include <linux/fs.h>
+#include <linux/acpi.h>
+#include <linux/vmalloc.h>
+#include <linux/pm_qos.h>
+#include <trace/events/power.h>
+
+#include <asm/cpu.h>
+#include <asm/div64.h>
+#include <asm/msr.h>
+#include <asm/cpu_device_id.h>
+#include <asm/cpufeature.h>
+#include <asm/intel-family.h>
+#include "../drivers/thermal/intel/thermal_interrupt.h"
+
+#define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
+
+#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
+#define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
+#define INTEL_CPUFREQ_TRANSITION_DELAY 500
+
+#ifdef CONFIG_ACPI
+#include <acpi/processor.h>
+#include <acpi/cppc_acpi.h>
+#endif
+
+#define FRAC_BITS 8
+#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
+#define fp_toint(X) ((X) >> FRAC_BITS)
+
+#define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3))
+
+#define EXT_BITS 6
+#define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
+#define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
+#define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
+
+static inline int32_t mul_fp(int32_t x, int32_t y)
+{
+ return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
+}
+
+static inline int32_t div_fp(s64 x, s64 y)
+{
+ return div64_s64((int64_t)x << FRAC_BITS, y);
+}
+
+static inline int ceiling_fp(int32_t x)
+{
+ int mask, ret;
+
+ ret = fp_toint(x);
+ mask = (1 << FRAC_BITS) - 1;
+ if (x & mask)
+ ret += 1;
+ return ret;
+}
+
+static inline u64 mul_ext_fp(u64 x, u64 y)
+{
+ return (x * y) >> EXT_FRAC_BITS;
+}
+
+static inline u64 div_ext_fp(u64 x, u64 y)
+{
+ return div64_u64(x << EXT_FRAC_BITS, y);
+}
+
+/**
+ * struct sample - Store performance sample
+ * @core_avg_perf: Ratio of APERF/MPERF which is the actual average
+ * performance during last sample period
+ * @busy_scaled: Scaled busy value which is used to calculate next
+ * P state. This can be different than core_avg_perf
+ * to account for cpu idle period
+ * @aperf: Difference of actual performance frequency clock count
+ * read from APERF MSR between last and current sample
+ * @mperf: Difference of maximum performance frequency clock count
+ * read from MPERF MSR between last and current sample
+ * @tsc: Difference of time stamp counter between last and
+ * current sample
+ * @time: Current time from scheduler
+ *
+ * This structure is used in the cpudata structure to store performance sample
+ * data for choosing next P State.
+ */
+struct sample {
+ int32_t core_avg_perf;
+ int32_t busy_scaled;
+ u64 aperf;
+ u64 mperf;
+ u64 tsc;
+ u64 time;
+};
+
+/**
+ * struct pstate_data - Store P state data
+ * @current_pstate: Current requested P state
+ * @min_pstate: Min P state possible for this platform
+ * @max_pstate: Max P state possible for this platform
+ * @max_pstate_physical:This is physical Max P state for a processor
+ * This can be higher than the max_pstate which can
+ * be limited by platform thermal design power limits
+ * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor
+ * @scaling: Scaling factor between performance and frequency
+ * @turbo_pstate: Max Turbo P state possible for this platform
+ * @min_freq: @min_pstate frequency in cpufreq units
+ * @max_freq: @max_pstate frequency in cpufreq units
+ * @turbo_freq: @turbo_pstate frequency in cpufreq units
+ *
+ * Stores the per cpu model P state limits and current P state.
+ */
+struct pstate_data {
+ int current_pstate;
+ int min_pstate;
+ int max_pstate;
+ int max_pstate_physical;
+ int perf_ctl_scaling;
+ int scaling;
+ int turbo_pstate;
+ unsigned int min_freq;
+ unsigned int max_freq;
+ unsigned int turbo_freq;
+};
+
+/**
+ * struct vid_data - Stores voltage information data
+ * @min: VID data for this platform corresponding to
+ * the lowest P state
+ * @max: VID data corresponding to the highest P State.
+ * @turbo: VID data for turbo P state
+ * @ratio: Ratio of (vid max - vid min) /
+ * (max P state - Min P State)
+ *
+ * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
+ * This data is used in Atom platforms, where in addition to target P state,
+ * the voltage data needs to be specified to select next P State.
+ */
+struct vid_data {
+ int min;
+ int max;
+ int turbo;
+ int32_t ratio;
+};
+
+/**
+ * struct global_params - Global parameters, mostly tunable via sysfs.
+ * @no_turbo: Whether or not to use turbo P-states.
+ * @turbo_disabled: Whether or not turbo P-states are available at all,
+ * based on the MSR_IA32_MISC_ENABLE value and whether or
+ * not the maximum reported turbo P-state is different from
+ * the maximum reported non-turbo one.
+ * @turbo_disabled_mf: The @turbo_disabled value reflected by cpuinfo.max_freq.
+ * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
+ * P-state capacity.
+ * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
+ * P-state capacity.
+ */
+struct global_params {
+ bool no_turbo;
+ bool turbo_disabled;
+ bool turbo_disabled_mf;
+ int max_perf_pct;
+ int min_perf_pct;
+};
+
+/**
+ * struct cpudata - Per CPU instance data storage
+ * @cpu: CPU number for this instance data
+ * @policy: CPUFreq policy value
+ * @update_util: CPUFreq utility callback information
+ * @update_util_set: CPUFreq utility callback is set
+ * @iowait_boost: iowait-related boost fraction
+ * @last_update: Time of the last update.
+ * @pstate: Stores P state limits for this CPU
+ * @vid: Stores VID limits for this CPU
+ * @last_sample_time: Last Sample time
+ * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
+ * @prev_aperf: Last APERF value read from APERF MSR
+ * @prev_mperf: Last MPERF value read from MPERF MSR
+ * @prev_tsc: Last timestamp counter (TSC) value
+ * @prev_cummulative_iowait: IO Wait time difference from last and
+ * current sample
+ * @sample: Storage for storing last Sample data
+ * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios
+ * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios
+ * @acpi_perf_data: Stores ACPI perf information read from _PSS
+ * @valid_pss_table: Set to true for valid ACPI _PSS entries found
+ * @epp_powersave: Last saved HWP energy performance preference
+ * (EPP) or energy performance bias (EPB),
+ * when policy switched to performance
+ * @epp_policy: Last saved policy used to set EPP/EPB
+ * @epp_default: Power on default HWP energy performance
+ * preference/bias
+ * @epp_cached Cached HWP energy-performance preference value
+ * @hwp_req_cached: Cached value of the last HWP Request MSR
+ * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
+ * @last_io_update: Last time when IO wake flag was set
+ * @sched_flags: Store scheduler flags for possible cross CPU update
+ * @hwp_boost_min: Last HWP boosted min performance
+ * @suspended: Whether or not the driver has been suspended.
+ * @hwp_notify_work: workqueue for HWP notifications.
+ *
+ * This structure stores per CPU instance data for all CPUs.
+ */
+struct cpudata {
+ int cpu;
+
+ unsigned int policy;
+ struct update_util_data update_util;
+ bool update_util_set;
+
+ struct pstate_data pstate;
+ struct vid_data vid;
+
+ u64 last_update;
+ u64 last_sample_time;
+ u64 aperf_mperf_shift;
+ u64 prev_aperf;
+ u64 prev_mperf;
+ u64 prev_tsc;
+ u64 prev_cummulative_iowait;
+ struct sample sample;
+ int32_t min_perf_ratio;
+ int32_t max_perf_ratio;
+#ifdef CONFIG_ACPI
+ struct acpi_processor_performance acpi_perf_data;
+ bool valid_pss_table;
+#endif
+ unsigned int iowait_boost;
+ s16 epp_powersave;
+ s16 epp_policy;
+ s16 epp_default;
+ s16 epp_cached;
+ u64 hwp_req_cached;
+ u64 hwp_cap_cached;
+ u64 last_io_update;
+ unsigned int sched_flags;
+ u32 hwp_boost_min;
+ bool suspended;
+ struct delayed_work hwp_notify_work;
+};
+
+static struct cpudata **all_cpu_data;
+
+/**
+ * struct pstate_funcs - Per CPU model specific callbacks
+ * @get_max: Callback to get maximum non turbo effective P state
+ * @get_max_physical: Callback to get maximum non turbo physical P state
+ * @get_min: Callback to get minimum P state
+ * @get_turbo: Callback to get turbo P state
+ * @get_scaling: Callback to get frequency scaling factor
+ * @get_cpu_scaling: Get frequency scaling factor for a given cpu
+ * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
+ * @get_val: Callback to convert P state to actual MSR write value
+ * @get_vid: Callback to get VID data for Atom platforms
+ *
+ * Core and Atom CPU models have different way to get P State limits. This
+ * structure is used to store those callbacks.
+ */
+struct pstate_funcs {
+ int (*get_max)(int cpu);
+ int (*get_max_physical)(int cpu);
+ int (*get_min)(int cpu);
+ int (*get_turbo)(int cpu);
+ int (*get_scaling)(void);
+ int (*get_cpu_scaling)(int cpu);
+ int (*get_aperf_mperf_shift)(void);
+ u64 (*get_val)(struct cpudata*, int pstate);
+ void (*get_vid)(struct cpudata *);
+};
+
+static struct pstate_funcs pstate_funcs __read_mostly;
+
+static int hwp_active __read_mostly;
+static int hwp_mode_bdw __read_mostly;
+static bool per_cpu_limits __read_mostly;
+static bool hwp_boost __read_mostly;
+static bool hwp_forced __read_mostly;
+
+static struct cpufreq_driver *intel_pstate_driver __read_mostly;
+
+#define HYBRID_SCALING_FACTOR 78741
+
+static inline int core_get_scaling(void)
+{
+ return 100000;
+}
+
+#ifdef CONFIG_ACPI
+static bool acpi_ppc;
+#endif
+
+static struct global_params global;
+
+static DEFINE_MUTEX(intel_pstate_driver_lock);
+static DEFINE_MUTEX(intel_pstate_limits_lock);
+
+#ifdef CONFIG_ACPI
+
+static bool intel_pstate_acpi_pm_profile_server(void)
+{
+ if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
+ acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
+ return true;
+
+ return false;
+}
+
+static bool intel_pstate_get_ppc_enable_status(void)
+{
+ if (intel_pstate_acpi_pm_profile_server())
+ return true;
+
+ return acpi_ppc;
+}
+
+#ifdef CONFIG_ACPI_CPPC_LIB
+
+/* The work item is needed to avoid CPU hotplug locking issues */
+static void intel_pstste_sched_itmt_work_fn(struct work_struct *work)
+{
+ sched_set_itmt_support();
+}
+
+static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn);
+
+#define CPPC_MAX_PERF U8_MAX
+
+static void intel_pstate_set_itmt_prio(int cpu)
+{
+ struct cppc_perf_caps cppc_perf;
+ static u32 max_highest_perf = 0, min_highest_perf = U32_MAX;
+ int ret;
+
+ ret = cppc_get_perf_caps(cpu, &cppc_perf);
+ if (ret)
+ return;
+
+ /*
+ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff.
+ * In this case we can't use CPPC.highest_perf to enable ITMT.
+ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide.
+ */
+ if (cppc_perf.highest_perf == CPPC_MAX_PERF)
+ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached));
+
+ /*
+ * The priorities can be set regardless of whether or not
+ * sched_set_itmt_support(true) has been called and it is valid to
+ * update them at any time after it has been called.
+ */
+ sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);
+
+ if (max_highest_perf <= min_highest_perf) {
+ if (cppc_perf.highest_perf > max_highest_perf)
+ max_highest_perf = cppc_perf.highest_perf;
+
+ if (cppc_perf.highest_perf < min_highest_perf)
+ min_highest_perf = cppc_perf.highest_perf;
+
+ if (max_highest_perf > min_highest_perf) {
+ /*
+ * This code can be run during CPU online under the
+ * CPU hotplug locks, so sched_set_itmt_support()
+ * cannot be called from here. Queue up a work item
+ * to invoke it.
+ */
+ schedule_work(&sched_itmt_work);
+ }
+ }
+}
+
+static int intel_pstate_get_cppc_guaranteed(int cpu)
+{
+ struct cppc_perf_caps cppc_perf;
+ int ret;
+
+ ret = cppc_get_perf_caps(cpu, &cppc_perf);
+ if (ret)
+ return ret;
+
+ if (cppc_perf.guaranteed_perf)
+ return cppc_perf.guaranteed_perf;
+
+ return cppc_perf.nominal_perf;
+}
+
+static int intel_pstate_cppc_get_scaling(int cpu)
+{
+ struct cppc_perf_caps cppc_perf;
+ int ret;
+
+ ret = cppc_get_perf_caps(cpu, &cppc_perf);
+
+ /*
+ * If the nominal frequency and the nominal performance are not
+ * zero and the ratio between them is not 100, return the hybrid
+ * scaling factor.
+ */
+ if (!ret && cppc_perf.nominal_perf && cppc_perf.nominal_freq &&
+ cppc_perf.nominal_perf * 100 != cppc_perf.nominal_freq)
+ return HYBRID_SCALING_FACTOR;
+
+ return core_get_scaling();
+}
+
+#else /* CONFIG_ACPI_CPPC_LIB */
+static inline void intel_pstate_set_itmt_prio(int cpu)
+{
+}
+#endif /* CONFIG_ACPI_CPPC_LIB */
+
+static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu;
+ int ret;
+ int i;
+
+ if (hwp_active) {
+ intel_pstate_set_itmt_prio(policy->cpu);
+ return;
+ }
+
+ if (!intel_pstate_get_ppc_enable_status())
+ return;
+
+ cpu = all_cpu_data[policy->cpu];
+
+ ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
+ policy->cpu);
+ if (ret)
+ return;
+
+ /*
+ * Check if the control value in _PSS is for PERF_CTL MSR, which should
+ * guarantee that the states returned by it map to the states in our
+ * list directly.
+ */
+ if (cpu->acpi_perf_data.control_register.space_id !=
+ ACPI_ADR_SPACE_FIXED_HARDWARE)
+ goto err;
+
+ /*
+ * If there is only one entry _PSS, simply ignore _PSS and continue as
+ * usual without taking _PSS into account
+ */
+ if (cpu->acpi_perf_data.state_count < 2)
+ goto err;
+
+ pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
+ for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
+ pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n",
+ (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
+ (u32) cpu->acpi_perf_data.states[i].core_frequency,
+ (u32) cpu->acpi_perf_data.states[i].power,
+ (u32) cpu->acpi_perf_data.states[i].control);
+ }
+
+ cpu->valid_pss_table = true;
+ pr_debug("_PPC limits will be enforced\n");
+
+ return;
+
+ err:
+ cpu->valid_pss_table = false;
+ acpi_processor_unregister_performance(policy->cpu);
+}
+
+static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu;
+
+ cpu = all_cpu_data[policy->cpu];
+ if (!cpu->valid_pss_table)
+ return;
+
+ acpi_processor_unregister_performance(policy->cpu);
+}
+#else /* CONFIG_ACPI */
+static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
+{
+}
+
+static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
+{
+}
+
+static inline bool intel_pstate_acpi_pm_profile_server(void)
+{
+ return false;
+}
+#endif /* CONFIG_ACPI */
+
+#ifndef CONFIG_ACPI_CPPC_LIB
+static inline int intel_pstate_get_cppc_guaranteed(int cpu)
+{
+ return -ENOTSUPP;
+}
+
+static int intel_pstate_cppc_get_scaling(int cpu)
+{
+ return core_get_scaling();
+}
+#endif /* CONFIG_ACPI_CPPC_LIB */
+
+static int intel_pstate_freq_to_hwp_rel(struct cpudata *cpu, int freq,
+ unsigned int relation)
+{
+ if (freq == cpu->pstate.turbo_freq)
+ return cpu->pstate.turbo_pstate;
+
+ if (freq == cpu->pstate.max_freq)
+ return cpu->pstate.max_pstate;
+
+ switch (relation) {
+ case CPUFREQ_RELATION_H:
+ return freq / cpu->pstate.scaling;
+ case CPUFREQ_RELATION_C:
+ return DIV_ROUND_CLOSEST(freq, cpu->pstate.scaling);
+ }
+
+ return DIV_ROUND_UP(freq, cpu->pstate.scaling);
+}
+
+static int intel_pstate_freq_to_hwp(struct cpudata *cpu, int freq)
+{
+ return intel_pstate_freq_to_hwp_rel(cpu, freq, CPUFREQ_RELATION_L);
+}
+
+/**
+ * intel_pstate_hybrid_hwp_adjust - Calibrate HWP performance levels.
+ * @cpu: Target CPU.
+ *
+ * On hybrid processors, HWP may expose more performance levels than there are
+ * P-states accessible through the PERF_CTL interface. If that happens, the
+ * scaling factor between HWP performance levels and CPU frequency will be less
+ * than the scaling factor between P-state values and CPU frequency.
+ *
+ * In that case, adjust the CPU parameters used in computations accordingly.
+ */
+static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu)
+{
+ int perf_ctl_max_phys = cpu->pstate.max_pstate_physical;
+ int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
+ int perf_ctl_turbo = pstate_funcs.get_turbo(cpu->cpu);
+ int scaling = cpu->pstate.scaling;
+ int freq;
+
+ pr_debug("CPU%d: perf_ctl_max_phys = %d\n", cpu->cpu, perf_ctl_max_phys);
+ pr_debug("CPU%d: perf_ctl_turbo = %d\n", cpu->cpu, perf_ctl_turbo);
+ pr_debug("CPU%d: perf_ctl_scaling = %d\n", cpu->cpu, perf_ctl_scaling);
+ pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate);
+ pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate);
+ pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling);
+
+ cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling,
+ perf_ctl_scaling);
+ cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling,
+ perf_ctl_scaling);
+
+ freq = perf_ctl_max_phys * perf_ctl_scaling;
+ cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq);
+
+ freq = cpu->pstate.min_pstate * perf_ctl_scaling;
+ cpu->pstate.min_freq = freq;
+ /*
+ * Cast the min P-state value retrieved via pstate_funcs.get_min() to
+ * the effective range of HWP performance levels.
+ */
+ cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq);
+}
+
+static inline void update_turbo_state(void)
+{
+ u64 misc_en;
+ struct cpudata *cpu;
+
+ cpu = all_cpu_data[0];
+ rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
+ global.turbo_disabled =
+ (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
+ cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
+}
+
+static int min_perf_pct_min(void)
+{
+ struct cpudata *cpu = all_cpu_data[0];
+ int turbo_pstate = cpu->pstate.turbo_pstate;
+
+ return turbo_pstate ?
+ (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0;
+}
+
+static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
+{
+ u64 epb;
+ int ret;
+
+ if (!boot_cpu_has(X86_FEATURE_EPB))
+ return -ENXIO;
+
+ ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
+ if (ret)
+ return (s16)ret;
+
+ return (s16)(epb & 0x0f);
+}
+
+static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)
+{
+ s16 epp;
+
+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
+ /*
+ * When hwp_req_data is 0, means that caller didn't read
+ * MSR_HWP_REQUEST, so need to read and get EPP.
+ */
+ if (!hwp_req_data) {
+ epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST,
+ &hwp_req_data);
+ if (epp)
+ return epp;
+ }
+ epp = (hwp_req_data >> 24) & 0xff;
+ } else {
+ /* When there is no EPP present, HWP uses EPB settings */
+ epp = intel_pstate_get_epb(cpu_data);
+ }
+
+ return epp;
+}
+
+static int intel_pstate_set_epb(int cpu, s16 pref)
+{
+ u64 epb;
+ int ret;
+
+ if (!boot_cpu_has(X86_FEATURE_EPB))
+ return -ENXIO;
+
+ ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
+ if (ret)
+ return ret;
+
+ epb = (epb & ~0x0f) | pref;
+ wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb);
+
+ return 0;
+}
+
+/*
+ * EPP/EPB display strings corresponding to EPP index in the
+ * energy_perf_strings[]
+ * index String
+ *-------------------------------------
+ * 0 default
+ * 1 performance
+ * 2 balance_performance
+ * 3 balance_power
+ * 4 power
+ */
+
+enum energy_perf_value_index {
+ EPP_INDEX_DEFAULT = 0,
+ EPP_INDEX_PERFORMANCE,
+ EPP_INDEX_BALANCE_PERFORMANCE,
+ EPP_INDEX_BALANCE_POWERSAVE,
+ EPP_INDEX_POWERSAVE,
+};
+
+static const char * const energy_perf_strings[] = {
+ [EPP_INDEX_DEFAULT] = "default",
+ [EPP_INDEX_PERFORMANCE] = "performance",
+ [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
+ [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
+ [EPP_INDEX_POWERSAVE] = "power",
+ NULL
+};
+static unsigned int epp_values[] = {
+ [EPP_INDEX_DEFAULT] = 0, /* Unused index */
+ [EPP_INDEX_PERFORMANCE] = HWP_EPP_PERFORMANCE,
+ [EPP_INDEX_BALANCE_PERFORMANCE] = HWP_EPP_BALANCE_PERFORMANCE,
+ [EPP_INDEX_BALANCE_POWERSAVE] = HWP_EPP_BALANCE_POWERSAVE,
+ [EPP_INDEX_POWERSAVE] = HWP_EPP_POWERSAVE,
+};
+
+static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)
+{
+ s16 epp;
+ int index = -EINVAL;
+
+ *raw_epp = 0;
+ epp = intel_pstate_get_epp(cpu_data, 0);
+ if (epp < 0)
+ return epp;
+
+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
+ if (epp == epp_values[EPP_INDEX_PERFORMANCE])
+ return EPP_INDEX_PERFORMANCE;
+ if (epp == epp_values[EPP_INDEX_BALANCE_PERFORMANCE])
+ return EPP_INDEX_BALANCE_PERFORMANCE;
+ if (epp == epp_values[EPP_INDEX_BALANCE_POWERSAVE])
+ return EPP_INDEX_BALANCE_POWERSAVE;
+ if (epp == epp_values[EPP_INDEX_POWERSAVE])
+ return EPP_INDEX_POWERSAVE;
+ *raw_epp = epp;
+ return 0;
+ } else if (boot_cpu_has(X86_FEATURE_EPB)) {
+ /*
+ * Range:
+ * 0x00-0x03 : Performance
+ * 0x04-0x07 : Balance performance
+ * 0x08-0x0B : Balance power
+ * 0x0C-0x0F : Power
+ * The EPB is a 4 bit value, but our ranges restrict the
+ * value which can be set. Here only using top two bits
+ * effectively.
+ */
+ index = (epp >> 2) + 1;
+ }
+
+ return index;
+}
+
+static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)
+{
+ int ret;
+
+ /*
+ * Use the cached HWP Request MSR value, because in the active mode the
+ * register itself may be updated by intel_pstate_hwp_boost_up() or
+ * intel_pstate_hwp_boost_down() at any time.
+ */
+ u64 value = READ_ONCE(cpu->hwp_req_cached);
+
+ value &= ~GENMASK_ULL(31, 24);
+ value |= (u64)epp << 24;
+ /*
+ * The only other updater of hwp_req_cached in the active mode,
+ * intel_pstate_hwp_set(), is called under the same lock as this
+ * function, so it cannot run in parallel with the update below.
+ */
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+ ret = wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
+ if (!ret)
+ cpu->epp_cached = epp;
+
+ return ret;
+}
+
+static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
+ int pref_index, bool use_raw,
+ u32 raw_epp)
+{
+ int epp = -EINVAL;
+ int ret;
+
+ if (!pref_index)
+ epp = cpu_data->epp_default;
+
+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
+ if (use_raw)
+ epp = raw_epp;
+ else if (epp == -EINVAL)
+ epp = epp_values[pref_index];
+
+ /*
+ * To avoid confusion, refuse to set EPP to any values different
+ * from 0 (performance) if the current policy is "performance",
+ * because those values would be overridden.
+ */
+ if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
+ return -EBUSY;
+
+ ret = intel_pstate_set_epp(cpu_data, epp);
+ } else {
+ if (epp == -EINVAL)
+ epp = (pref_index - 1) << 2;
+ ret = intel_pstate_set_epb(cpu_data->cpu, epp);
+ }
+
+ return ret;
+}
+
+static ssize_t show_energy_performance_available_preferences(
+ struct cpufreq_policy *policy, char *buf)
+{
+ int i = 0;
+ int ret = 0;
+
+ while (energy_perf_strings[i] != NULL)
+ ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]);
+
+ ret += sprintf(&buf[ret], "\n");
+
+ return ret;
+}
+
+cpufreq_freq_attr_ro(energy_performance_available_preferences);
+
+static struct cpufreq_driver intel_pstate;
+
+static ssize_t store_energy_performance_preference(
+ struct cpufreq_policy *policy, const char *buf, size_t count)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ char str_preference[21];
+ bool raw = false;
+ ssize_t ret;
+ u32 epp = 0;
+
+ ret = sscanf(buf, "%20s", str_preference);
+ if (ret != 1)
+ return -EINVAL;
+
+ ret = match_string(energy_perf_strings, -1, str_preference);
+ if (ret < 0) {
+ if (!boot_cpu_has(X86_FEATURE_HWP_EPP))
+ return ret;
+
+ ret = kstrtouint(buf, 10, &epp);
+ if (ret)
+ return ret;
+
+ if (epp > 255)
+ return -EINVAL;
+
+ raw = true;
+ }
+
+ /*
+ * This function runs with the policy R/W semaphore held, which
+ * guarantees that the driver pointer will not change while it is
+ * running.
+ */
+ if (!intel_pstate_driver)
+ return -EAGAIN;
+
+ mutex_lock(&intel_pstate_limits_lock);
+
+ if (intel_pstate_driver == &intel_pstate) {
+ ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp);
+ } else {
+ /*
+ * In the passive mode the governor needs to be stopped on the
+ * target CPU before the EPP update and restarted after it,
+ * which is super-heavy-weight, so make sure it is worth doing
+ * upfront.
+ */
+ if (!raw)
+ epp = ret ? epp_values[ret] : cpu->epp_default;
+
+ if (cpu->epp_cached != epp) {
+ int err;
+
+ cpufreq_stop_governor(policy);
+ ret = intel_pstate_set_epp(cpu, epp);
+ err = cpufreq_start_governor(policy);
+ if (!ret)
+ ret = err;
+ } else {
+ ret = 0;
+ }
+ }
+
+ mutex_unlock(&intel_pstate_limits_lock);
+
+ return ret ?: count;
+}
+
+static ssize_t show_energy_performance_preference(
+ struct cpufreq_policy *policy, char *buf)
+{
+ struct cpudata *cpu_data = all_cpu_data[policy->cpu];
+ int preference, raw_epp;
+
+ preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp);
+ if (preference < 0)
+ return preference;
+
+ if (raw_epp)
+ return sprintf(buf, "%d\n", raw_epp);
+ else
+ return sprintf(buf, "%s\n", energy_perf_strings[preference]);
+}
+
+cpufreq_freq_attr_rw(energy_performance_preference);
+
+static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ int ratio, freq;
+
+ ratio = intel_pstate_get_cppc_guaranteed(policy->cpu);
+ if (ratio <= 0) {
+ u64 cap;
+
+ rdmsrl_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap);
+ ratio = HWP_GUARANTEED_PERF(cap);
+ }
+
+ freq = ratio * cpu->pstate.scaling;
+ if (cpu->pstate.scaling != cpu->pstate.perf_ctl_scaling)
+ freq = rounddown(freq, cpu->pstate.perf_ctl_scaling);
+
+ return sprintf(buf, "%d\n", freq);
+}
+
+cpufreq_freq_attr_ro(base_frequency);
+
+static struct freq_attr *hwp_cpufreq_attrs[] = {
+ &energy_performance_preference,
+ &energy_performance_available_preferences,
+ &base_frequency,
+ NULL,
+};
+
+static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)
+{
+ u64 cap;
+
+ rdmsrl_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap);
+ WRITE_ONCE(cpu->hwp_cap_cached, cap);
+ cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(cap);
+ cpu->pstate.turbo_pstate = HWP_HIGHEST_PERF(cap);
+}
+
+static void intel_pstate_get_hwp_cap(struct cpudata *cpu)
+{
+ int scaling = cpu->pstate.scaling;
+
+ __intel_pstate_get_hwp_cap(cpu);
+
+ cpu->pstate.max_freq = cpu->pstate.max_pstate * scaling;
+ cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling;
+ if (scaling != cpu->pstate.perf_ctl_scaling) {
+ int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
+
+ cpu->pstate.max_freq = rounddown(cpu->pstate.max_freq,
+ perf_ctl_scaling);
+ cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_freq,
+ perf_ctl_scaling);
+ }
+}
+
+static void intel_pstate_hwp_set(unsigned int cpu)
+{
+ struct cpudata *cpu_data = all_cpu_data[cpu];
+ int max, min;
+ u64 value;
+ s16 epp;
+
+ max = cpu_data->max_perf_ratio;
+ min = cpu_data->min_perf_ratio;
+
+ if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
+ min = max;
+
+ rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
+
+ value &= ~HWP_MIN_PERF(~0L);
+ value |= HWP_MIN_PERF(min);
+
+ value &= ~HWP_MAX_PERF(~0L);
+ value |= HWP_MAX_PERF(max);
+
+ if (cpu_data->epp_policy == cpu_data->policy)
+ goto skip_epp;
+
+ cpu_data->epp_policy = cpu_data->policy;
+
+ if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
+ epp = intel_pstate_get_epp(cpu_data, value);
+ cpu_data->epp_powersave = epp;
+ /* If EPP read was failed, then don't try to write */
+ if (epp < 0)
+ goto skip_epp;
+
+ epp = 0;
+ } else {
+ /* skip setting EPP, when saved value is invalid */
+ if (cpu_data->epp_powersave < 0)
+ goto skip_epp;
+
+ /*
+ * No need to restore EPP when it is not zero. This
+ * means:
+ * - Policy is not changed
+ * - user has manually changed
+ * - Error reading EPB
+ */
+ epp = intel_pstate_get_epp(cpu_data, value);
+ if (epp)
+ goto skip_epp;
+
+ epp = cpu_data->epp_powersave;
+ }
+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
+ value &= ~GENMASK_ULL(31, 24);
+ value |= (u64)epp << 24;
+ } else {
+ intel_pstate_set_epb(cpu, epp);
+ }
+skip_epp:
+ WRITE_ONCE(cpu_data->hwp_req_cached, value);
+ wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
+}
+
+static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata);
+
+static void intel_pstate_hwp_offline(struct cpudata *cpu)
+{
+ u64 value = READ_ONCE(cpu->hwp_req_cached);
+ int min_perf;
+
+ intel_pstate_disable_hwp_interrupt(cpu);
+
+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
+ /*
+ * In case the EPP has been set to "performance" by the
+ * active mode "performance" scaling algorithm, replace that
+ * temporary value with the cached EPP one.
+ */
+ value &= ~GENMASK_ULL(31, 24);
+ value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached);
+ /*
+ * However, make sure that EPP will be set to "performance" when
+ * the CPU is brought back online again and the "performance"
+ * scaling algorithm is still in effect.
+ */
+ cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN;
+ }
+
+ /*
+ * Clear the desired perf field in the cached HWP request value to
+ * prevent nonzero desired values from being leaked into the active
+ * mode.
+ */
+ value &= ~HWP_DESIRED_PERF(~0L);
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+
+ value &= ~GENMASK_ULL(31, 0);
+ min_perf = HWP_LOWEST_PERF(READ_ONCE(cpu->hwp_cap_cached));
+
+ /* Set hwp_max = hwp_min */
+ value |= HWP_MAX_PERF(min_perf);
+ value |= HWP_MIN_PERF(min_perf);
+
+ /* Set EPP to min */
+ if (boot_cpu_has(X86_FEATURE_HWP_EPP))
+ value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE);
+
+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
+}
+
+#define POWER_CTL_EE_ENABLE 1
+#define POWER_CTL_EE_DISABLE 2
+
+static int power_ctl_ee_state;
+
+static void set_power_ctl_ee_state(bool input)
+{
+ u64 power_ctl;
+
+ mutex_lock(&intel_pstate_driver_lock);
+ rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
+ if (input) {
+ power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE);
+ power_ctl_ee_state = POWER_CTL_EE_ENABLE;
+ } else {
+ power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
+ power_ctl_ee_state = POWER_CTL_EE_DISABLE;
+ }
+ wrmsrl(MSR_IA32_POWER_CTL, power_ctl);
+ mutex_unlock(&intel_pstate_driver_lock);
+}
+
+static void intel_pstate_hwp_enable(struct cpudata *cpudata);
+
+static void intel_pstate_hwp_reenable(struct cpudata *cpu)
+{
+ intel_pstate_hwp_enable(cpu);
+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached));
+}
+
+static int intel_pstate_suspend(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ pr_debug("CPU %d suspending\n", cpu->cpu);
+
+ cpu->suspended = true;
+
+ /* disable HWP interrupt and cancel any pending work */
+ intel_pstate_disable_hwp_interrupt(cpu);
+
+ return 0;
+}
+
+static int intel_pstate_resume(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ pr_debug("CPU %d resuming\n", cpu->cpu);
+
+ /* Only restore if the system default is changed */
+ if (power_ctl_ee_state == POWER_CTL_EE_ENABLE)
+ set_power_ctl_ee_state(true);
+ else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE)
+ set_power_ctl_ee_state(false);
+
+ if (cpu->suspended && hwp_active) {
+ mutex_lock(&intel_pstate_limits_lock);
+
+ /* Re-enable HWP, because "online" has not done that. */
+ intel_pstate_hwp_reenable(cpu);
+
+ mutex_unlock(&intel_pstate_limits_lock);
+ }
+
+ cpu->suspended = false;
+
+ return 0;
+}
+
+static void intel_pstate_update_policies(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ cpufreq_update_policy(cpu);
+}
+
+static void __intel_pstate_update_max_freq(struct cpudata *cpudata,
+ struct cpufreq_policy *policy)
+{
+ policy->cpuinfo.max_freq = global.turbo_disabled_mf ?
+ cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;
+ refresh_frequency_limits(policy);
+}
+
+static void intel_pstate_update_max_freq(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
+
+ if (!policy)
+ return;
+
+ __intel_pstate_update_max_freq(all_cpu_data[cpu], policy);
+
+ cpufreq_cpu_release(policy);
+}
+
+static void intel_pstate_update_limits(unsigned int cpu)
+{
+ mutex_lock(&intel_pstate_driver_lock);
+
+ update_turbo_state();
+ /*
+ * If turbo has been turned on or off globally, policy limits for
+ * all CPUs need to be updated to reflect that.
+ */
+ if (global.turbo_disabled_mf != global.turbo_disabled) {
+ global.turbo_disabled_mf = global.turbo_disabled;
+ arch_set_max_freq_ratio(global.turbo_disabled);
+ for_each_possible_cpu(cpu)
+ intel_pstate_update_max_freq(cpu);
+ } else {
+ cpufreq_update_policy(cpu);
+ }
+
+ mutex_unlock(&intel_pstate_driver_lock);
+}
+
+/************************** sysfs begin ************************/
+#define show_one(file_name, object) \
+ static ssize_t show_##file_name \
+ (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
+ { \
+ return sprintf(buf, "%u\n", global.object); \
+ }
+
+static ssize_t intel_pstate_show_status(char *buf);
+static int intel_pstate_update_status(const char *buf, size_t size);
+
+static ssize_t show_status(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ ssize_t ret;
+
+ mutex_lock(&intel_pstate_driver_lock);
+ ret = intel_pstate_show_status(buf);
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return ret;
+}
+
+static ssize_t store_status(struct kobject *a, struct kobj_attribute *b,
+ const char *buf, size_t count)
+{
+ char *p = memchr(buf, '\n', count);
+ int ret;
+
+ mutex_lock(&intel_pstate_driver_lock);
+ ret = intel_pstate_update_status(buf, p ? p - buf : count);
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return ret < 0 ? ret : count;
+}
+
+static ssize_t show_turbo_pct(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct cpudata *cpu;
+ int total, no_turbo, turbo_pct;
+ uint32_t turbo_fp;
+
+ mutex_lock(&intel_pstate_driver_lock);
+
+ if (!intel_pstate_driver) {
+ mutex_unlock(&intel_pstate_driver_lock);
+ return -EAGAIN;
+ }
+
+ cpu = all_cpu_data[0];
+
+ total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
+ no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
+ turbo_fp = div_fp(no_turbo, total);
+ turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
+
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return sprintf(buf, "%u\n", turbo_pct);
+}
+
+static ssize_t show_num_pstates(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ struct cpudata *cpu;
+ int total;
+
+ mutex_lock(&intel_pstate_driver_lock);
+
+ if (!intel_pstate_driver) {
+ mutex_unlock(&intel_pstate_driver_lock);
+ return -EAGAIN;
+ }
+
+ cpu = all_cpu_data[0];
+ total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
+
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return sprintf(buf, "%u\n", total);
+}
+
+static ssize_t show_no_turbo(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ ssize_t ret;
+
+ mutex_lock(&intel_pstate_driver_lock);
+
+ if (!intel_pstate_driver) {
+ mutex_unlock(&intel_pstate_driver_lock);
+ return -EAGAIN;
+ }
+
+ update_turbo_state();
+ if (global.turbo_disabled)
+ ret = sprintf(buf, "%u\n", global.turbo_disabled);
+ else
+ ret = sprintf(buf, "%u\n", global.no_turbo);
+
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return ret;
+}
+
+static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b,
+ const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+
+ mutex_lock(&intel_pstate_driver_lock);
+
+ if (!intel_pstate_driver) {
+ mutex_unlock(&intel_pstate_driver_lock);
+ return -EAGAIN;
+ }
+
+ mutex_lock(&intel_pstate_limits_lock);
+
+ update_turbo_state();
+ if (global.turbo_disabled) {
+ pr_notice_once("Turbo disabled by BIOS or unavailable on processor\n");
+ mutex_unlock(&intel_pstate_limits_lock);
+ mutex_unlock(&intel_pstate_driver_lock);
+ return -EPERM;
+ }
+
+ global.no_turbo = clamp_t(int, input, 0, 1);
+
+ if (global.no_turbo) {
+ struct cpudata *cpu = all_cpu_data[0];
+ int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
+
+ /* Squash the global minimum into the permitted range. */
+ if (global.min_perf_pct > pct)
+ global.min_perf_pct = pct;
+ }
+
+ mutex_unlock(&intel_pstate_limits_lock);
+
+ intel_pstate_update_policies();
+ arch_set_max_freq_ratio(global.no_turbo);
+
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return count;
+}
+
+static void update_qos_request(enum freq_qos_req_type type)
+{
+ struct freq_qos_request *req;
+ struct cpufreq_policy *policy;
+ int i;
+
+ for_each_possible_cpu(i) {
+ struct cpudata *cpu = all_cpu_data[i];
+ unsigned int freq, perf_pct;
+
+ policy = cpufreq_cpu_get(i);
+ if (!policy)
+ continue;
+
+ req = policy->driver_data;
+ cpufreq_cpu_put(policy);
+
+ if (!req)
+ continue;
+
+ if (hwp_active)
+ intel_pstate_get_hwp_cap(cpu);
+
+ if (type == FREQ_QOS_MIN) {
+ perf_pct = global.min_perf_pct;
+ } else {
+ req++;
+ perf_pct = global.max_perf_pct;
+ }
+
+ freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * perf_pct, 100);
+
+ if (freq_qos_update_request(req, freq) < 0)
+ pr_warn("Failed to update freq constraint: CPU%d\n", i);
+ }
+}
+
+static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b,
+ const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+
+ mutex_lock(&intel_pstate_driver_lock);
+
+ if (!intel_pstate_driver) {
+ mutex_unlock(&intel_pstate_driver_lock);
+ return -EAGAIN;
+ }
+
+ mutex_lock(&intel_pstate_limits_lock);
+
+ global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
+
+ mutex_unlock(&intel_pstate_limits_lock);
+
+ if (intel_pstate_driver == &intel_pstate)
+ intel_pstate_update_policies();
+ else
+ update_qos_request(FREQ_QOS_MAX);
+
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return count;
+}
+
+static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b,
+ const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+
+ ret = sscanf(buf, "%u", &input);
+ if (ret != 1)
+ return -EINVAL;
+
+ mutex_lock(&intel_pstate_driver_lock);
+
+ if (!intel_pstate_driver) {
+ mutex_unlock(&intel_pstate_driver_lock);
+ return -EAGAIN;
+ }
+
+ mutex_lock(&intel_pstate_limits_lock);
+
+ global.min_perf_pct = clamp_t(int, input,
+ min_perf_pct_min(), global.max_perf_pct);
+
+ mutex_unlock(&intel_pstate_limits_lock);
+
+ if (intel_pstate_driver == &intel_pstate)
+ intel_pstate_update_policies();
+ else
+ update_qos_request(FREQ_QOS_MIN);
+
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return count;
+}
+
+static ssize_t show_hwp_dynamic_boost(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ return sprintf(buf, "%u\n", hwp_boost);
+}
+
+static ssize_t store_hwp_dynamic_boost(struct kobject *a,
+ struct kobj_attribute *b,
+ const char *buf, size_t count)
+{
+ unsigned int input;
+ int ret;
+
+ ret = kstrtouint(buf, 10, &input);
+ if (ret)
+ return ret;
+
+ mutex_lock(&intel_pstate_driver_lock);
+ hwp_boost = !!input;
+ intel_pstate_update_policies();
+ mutex_unlock(&intel_pstate_driver_lock);
+
+ return count;
+}
+
+static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf)
+{
+ u64 power_ctl;
+ int enable;
+
+ rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
+ enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE));
+ return sprintf(buf, "%d\n", !enable);
+}
+
+static ssize_t store_energy_efficiency(struct kobject *a, struct kobj_attribute *b,
+ const char *buf, size_t count)
+{
+ bool input;
+ int ret;
+
+ ret = kstrtobool(buf, &input);
+ if (ret)
+ return ret;
+
+ set_power_ctl_ee_state(input);
+
+ return count;
+}
+
+show_one(max_perf_pct, max_perf_pct);
+show_one(min_perf_pct, min_perf_pct);
+
+define_one_global_rw(status);
+define_one_global_rw(no_turbo);
+define_one_global_rw(max_perf_pct);
+define_one_global_rw(min_perf_pct);
+define_one_global_ro(turbo_pct);
+define_one_global_ro(num_pstates);
+define_one_global_rw(hwp_dynamic_boost);
+define_one_global_rw(energy_efficiency);
+
+static struct attribute *intel_pstate_attributes[] = {
+ &status.attr,
+ &no_turbo.attr,
+ NULL
+};
+
+static const struct attribute_group intel_pstate_attr_group = {
+ .attrs = intel_pstate_attributes,
+};
+
+static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[];
+
+static struct kobject *intel_pstate_kobject;
+
+static void __init intel_pstate_sysfs_expose_params(void)
+{
+ struct device *dev_root = bus_get_dev_root(&cpu_subsys);
+ int rc;
+
+ if (dev_root) {
+ intel_pstate_kobject = kobject_create_and_add("intel_pstate", &dev_root->kobj);
+ put_device(dev_root);
+ }
+ if (WARN_ON(!intel_pstate_kobject))
+ return;
+
+ rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
+ if (WARN_ON(rc))
+ return;
+
+ if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
+ rc = sysfs_create_file(intel_pstate_kobject, &turbo_pct.attr);
+ WARN_ON(rc);
+
+ rc = sysfs_create_file(intel_pstate_kobject, &num_pstates.attr);
+ WARN_ON(rc);
+ }
+
+ /*
+ * If per cpu limits are enforced there are no global limits, so
+ * return without creating max/min_perf_pct attributes
+ */
+ if (per_cpu_limits)
+ return;
+
+ rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
+ WARN_ON(rc);
+
+ rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
+ WARN_ON(rc);
+
+ if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) {
+ rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr);
+ WARN_ON(rc);
+ }
+}
+
+static void __init intel_pstate_sysfs_remove(void)
+{
+ if (!intel_pstate_kobject)
+ return;
+
+ sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group);
+
+ if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
+ sysfs_remove_file(intel_pstate_kobject, &num_pstates.attr);
+ sysfs_remove_file(intel_pstate_kobject, &turbo_pct.attr);
+ }
+
+ if (!per_cpu_limits) {
+ sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr);
+ sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr);
+
+ if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids))
+ sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr);
+ }
+
+ kobject_put(intel_pstate_kobject);
+}
+
+static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
+{
+ int rc;
+
+ if (!hwp_active)
+ return;
+
+ rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
+ WARN_ON_ONCE(rc);
+}
+
+static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
+{
+ if (!hwp_active)
+ return;
+
+ sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
+}
+
+/************************** sysfs end ************************/
+
+static void intel_pstate_notify_work(struct work_struct *work)
+{
+ struct cpudata *cpudata =
+ container_of(to_delayed_work(work), struct cpudata, hwp_notify_work);
+ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpudata->cpu);
+
+ if (policy) {
+ intel_pstate_get_hwp_cap(cpudata);
+ __intel_pstate_update_max_freq(cpudata, policy);
+
+ cpufreq_cpu_release(policy);
+ }
+
+ wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);
+}
+
+static DEFINE_SPINLOCK(hwp_notify_lock);
+static cpumask_t hwp_intr_enable_mask;
+
+void notify_hwp_interrupt(void)
+{
+ unsigned int this_cpu = smp_processor_id();
+ struct cpudata *cpudata;
+ unsigned long flags;
+ u64 value;
+
+ if (!READ_ONCE(hwp_active) || !boot_cpu_has(X86_FEATURE_HWP_NOTIFY))
+ return;
+
+ rdmsrl_safe(MSR_HWP_STATUS, &value);
+ if (!(value & 0x01))
+ return;
+
+ spin_lock_irqsave(&hwp_notify_lock, flags);
+
+ if (!cpumask_test_cpu(this_cpu, &hwp_intr_enable_mask))
+ goto ack_intr;
+
+ /*
+ * Currently we never free all_cpu_data. And we can't reach here
+ * without this allocated. But for safety for future changes, added
+ * check.
+ */
+ if (unlikely(!READ_ONCE(all_cpu_data)))
+ goto ack_intr;
+
+ /*
+ * The free is done during cleanup, when cpufreq registry is failed.
+ * We wouldn't be here if it fails on init or switch status. But for
+ * future changes, added check.
+ */
+ cpudata = READ_ONCE(all_cpu_data[this_cpu]);
+ if (unlikely(!cpudata))
+ goto ack_intr;
+
+ schedule_delayed_work(&cpudata->hwp_notify_work, msecs_to_jiffies(10));
+
+ spin_unlock_irqrestore(&hwp_notify_lock, flags);
+
+ return;
+
+ack_intr:
+ wrmsrl_safe(MSR_HWP_STATUS, 0);
+ spin_unlock_irqrestore(&hwp_notify_lock, flags);
+}
+
+static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata)
+{
+ unsigned long flags;
+
+ if (!boot_cpu_has(X86_FEATURE_HWP_NOTIFY))
+ return;
+
+ /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */
+ wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
+
+ spin_lock_irqsave(&hwp_notify_lock, flags);
+ if (cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask))
+ cancel_delayed_work(&cpudata->hwp_notify_work);
+ spin_unlock_irqrestore(&hwp_notify_lock, flags);
+}
+
+static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata)
+{
+ /* Enable HWP notification interrupt for guaranteed performance change */
+ if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&hwp_notify_lock, flags);
+ INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work);
+ cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask);
+ spin_unlock_irqrestore(&hwp_notify_lock, flags);
+
+ /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */
+ wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x01);
+ wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);
+ }
+}
+
+static void intel_pstate_update_epp_defaults(struct cpudata *cpudata)
+{
+ cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
+
+ /*
+ * If this CPU gen doesn't call for change in balance_perf
+ * EPP return.
+ */
+ if (epp_values[EPP_INDEX_BALANCE_PERFORMANCE] == HWP_EPP_BALANCE_PERFORMANCE)
+ return;
+
+ /*
+ * If the EPP is set by firmware, which means that firmware enabled HWP
+ * - Is equal or less than 0x80 (default balance_perf EPP)
+ * - But less performance oriented than performance EPP
+ * then use this as new balance_perf EPP.
+ */
+ if (hwp_forced && cpudata->epp_default <= HWP_EPP_BALANCE_PERFORMANCE &&
+ cpudata->epp_default > HWP_EPP_PERFORMANCE) {
+ epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default;
+ return;
+ }
+
+ /*
+ * Use hard coded value per gen to update the balance_perf
+ * and default EPP.
+ */
+ cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE];
+ intel_pstate_set_epp(cpudata, cpudata->epp_default);
+}
+
+static void intel_pstate_hwp_enable(struct cpudata *cpudata)
+{
+ /* First disable HWP notification interrupt till we activate again */
+ if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY))
+ wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
+
+ wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
+
+ intel_pstate_enable_hwp_interrupt(cpudata);
+
+ if (cpudata->epp_default >= 0)
+ return;
+
+ intel_pstate_update_epp_defaults(cpudata);
+}
+
+static int atom_get_min_pstate(int not_used)
+{
+ u64 value;
+
+ rdmsrl(MSR_ATOM_CORE_RATIOS, value);
+ return (value >> 8) & 0x7F;
+}
+
+static int atom_get_max_pstate(int not_used)
+{
+ u64 value;
+
+ rdmsrl(MSR_ATOM_CORE_RATIOS, value);
+ return (value >> 16) & 0x7F;
+}
+
+static int atom_get_turbo_pstate(int not_used)
+{
+ u64 value;
+
+ rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value);
+ return value & 0x7F;
+}
+
+static u64 atom_get_val(struct cpudata *cpudata, int pstate)
+{
+ u64 val;
+ int32_t vid_fp;
+ u32 vid;
+
+ val = (u64)pstate << 8;
+ if (global.no_turbo && !global.turbo_disabled)
+ val |= (u64)1 << 32;
+
+ vid_fp = cpudata->vid.min + mul_fp(
+ int_tofp(pstate - cpudata->pstate.min_pstate),
+ cpudata->vid.ratio);
+
+ vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
+ vid = ceiling_fp(vid_fp);
+
+ if (pstate > cpudata->pstate.max_pstate)
+ vid = cpudata->vid.turbo;
+
+ return val | vid;
+}
+
+static int silvermont_get_scaling(void)
+{
+ u64 value;
+ int i;
+ /* Defined in Table 35-6 from SDM (Sept 2015) */
+ static int silvermont_freq_table[] = {
+ 83300, 100000, 133300, 116700, 80000};
+
+ rdmsrl(MSR_FSB_FREQ, value);
+ i = value & 0x7;
+ WARN_ON(i > 4);
+
+ return silvermont_freq_table[i];
+}
+
+static int airmont_get_scaling(void)
+{
+ u64 value;
+ int i;
+ /* Defined in Table 35-10 from SDM (Sept 2015) */
+ static int airmont_freq_table[] = {
+ 83300, 100000, 133300, 116700, 80000,
+ 93300, 90000, 88900, 87500};
+
+ rdmsrl(MSR_FSB_FREQ, value);
+ i = value & 0xF;
+ WARN_ON(i > 8);
+
+ return airmont_freq_table[i];
+}
+
+static void atom_get_vid(struct cpudata *cpudata)
+{
+ u64 value;
+
+ rdmsrl(MSR_ATOM_CORE_VIDS, value);
+ cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
+ cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
+ cpudata->vid.ratio = div_fp(
+ cpudata->vid.max - cpudata->vid.min,
+ int_tofp(cpudata->pstate.max_pstate -
+ cpudata->pstate.min_pstate));
+
+ rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value);
+ cpudata->vid.turbo = value & 0x7f;
+}
+
+static int core_get_min_pstate(int cpu)
+{
+ u64 value;
+
+ rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value);
+ return (value >> 40) & 0xFF;
+}
+
+static int core_get_max_pstate_physical(int cpu)
+{
+ u64 value;
+
+ rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value);
+ return (value >> 8) & 0xFF;
+}
+
+static int core_get_tdp_ratio(int cpu, u64 plat_info)
+{
+ /* Check how many TDP levels present */
+ if (plat_info & 0x600000000) {
+ u64 tdp_ctrl;
+ u64 tdp_ratio;
+ int tdp_msr;
+ int err;
+
+ /* Get the TDP level (0, 1, 2) to get ratios */
+ err = rdmsrl_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
+ if (err)
+ return err;
+
+ /* TDP MSR are continuous starting at 0x648 */
+ tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03);
+ err = rdmsrl_safe_on_cpu(cpu, tdp_msr, &tdp_ratio);
+ if (err)
+ return err;
+
+ /* For level 1 and 2, bits[23:16] contain the ratio */
+ if (tdp_ctrl & 0x03)
+ tdp_ratio >>= 16;
+
+ tdp_ratio &= 0xff; /* ratios are only 8 bits long */
+ pr_debug("tdp_ratio %x\n", (int)tdp_ratio);
+
+ return (int)tdp_ratio;
+ }
+
+ return -ENXIO;
+}
+
+static int core_get_max_pstate(int cpu)
+{
+ u64 tar;
+ u64 plat_info;
+ int max_pstate;
+ int tdp_ratio;
+ int err;
+
+ rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info);
+ max_pstate = (plat_info >> 8) & 0xFF;
+
+ tdp_ratio = core_get_tdp_ratio(cpu, plat_info);
+ if (tdp_ratio <= 0)
+ return max_pstate;
+
+ if (hwp_active) {
+ /* Turbo activation ratio is not used on HWP platforms */
+ return tdp_ratio;
+ }
+
+ err = rdmsrl_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar);
+ if (!err) {
+ int tar_levels;
+
+ /* Do some sanity checking for safety */
+ tar_levels = tar & 0xff;
+ if (tdp_ratio - 1 == tar_levels) {
+ max_pstate = tar_levels;
+ pr_debug("max_pstate=TAC %x\n", max_pstate);
+ }
+ }
+
+ return max_pstate;
+}
+
+static int core_get_turbo_pstate(int cpu)
+{
+ u64 value;
+ int nont, ret;
+
+ rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value);
+ nont = core_get_max_pstate(cpu);
+ ret = (value) & 255;
+ if (ret <= nont)
+ ret = nont;
+ return ret;
+}
+
+static u64 core_get_val(struct cpudata *cpudata, int pstate)
+{
+ u64 val;
+
+ val = (u64)pstate << 8;
+ if (global.no_turbo && !global.turbo_disabled)
+ val |= (u64)1 << 32;
+
+ return val;
+}
+
+static int knl_get_aperf_mperf_shift(void)
+{
+ return 10;
+}
+
+static int knl_get_turbo_pstate(int cpu)
+{
+ u64 value;
+ int nont, ret;
+
+ rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value);
+ nont = core_get_max_pstate(cpu);
+ ret = (((value) >> 8) & 0xFF);
+ if (ret <= nont)
+ ret = nont;
+ return ret;
+}
+
+static void hybrid_get_type(void *data)
+{
+ u8 *cpu_type = data;
+
+ *cpu_type = get_this_hybrid_cpu_type();
+}
+
+static int hwp_get_cpu_scaling(int cpu)
+{
+ u8 cpu_type = 0;
+
+ smp_call_function_single(cpu, hybrid_get_type, &cpu_type, 1);
+ /* P-cores have a smaller perf level-to-freqency scaling factor. */
+ if (cpu_type == 0x40)
+ return HYBRID_SCALING_FACTOR;
+
+ /* Use default core scaling for E-cores */
+ if (cpu_type == 0x20)
+ return core_get_scaling();
+
+ /*
+ * If reached here, this system is either non-hybrid (like Tiger
+ * Lake) or hybrid-capable (like Alder Lake or Raptor Lake) with
+ * no E cores (in which case CPUID for hybrid support is 0).
+ *
+ * The CPPC nominal_frequency field is 0 for non-hybrid systems,
+ * so the default core scaling will be used for them.
+ */
+ return intel_pstate_cppc_get_scaling(cpu);
+}
+
+static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
+{
+ trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
+ cpu->pstate.current_pstate = pstate;
+ /*
+ * Generally, there is no guarantee that this code will always run on
+ * the CPU being updated, so force the register update to run on the
+ * right CPU.
+ */
+ wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
+ pstate_funcs.get_val(cpu, pstate));
+}
+
+static void intel_pstate_set_min_pstate(struct cpudata *cpu)
+{
+ intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
+}
+
+static void intel_pstate_max_within_limits(struct cpudata *cpu)
+{
+ int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio);
+
+ update_turbo_state();
+ intel_pstate_set_pstate(cpu, pstate);
+}
+
+static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
+{
+ int perf_ctl_max_phys = pstate_funcs.get_max_physical(cpu->cpu);
+ int perf_ctl_scaling = pstate_funcs.get_scaling();
+
+ cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu);
+ cpu->pstate.max_pstate_physical = perf_ctl_max_phys;
+ cpu->pstate.perf_ctl_scaling = perf_ctl_scaling;
+
+ if (hwp_active && !hwp_mode_bdw) {
+ __intel_pstate_get_hwp_cap(cpu);
+
+ if (pstate_funcs.get_cpu_scaling) {
+ cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpu->cpu);
+ if (cpu->pstate.scaling != perf_ctl_scaling)
+ intel_pstate_hybrid_hwp_adjust(cpu);
+ } else {
+ cpu->pstate.scaling = perf_ctl_scaling;
+ }
+ } else {
+ cpu->pstate.scaling = perf_ctl_scaling;
+ cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu);
+ cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu);
+ }
+
+ if (cpu->pstate.scaling == perf_ctl_scaling) {
+ cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling;
+ cpu->pstate.max_freq = cpu->pstate.max_pstate * perf_ctl_scaling;
+ cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * perf_ctl_scaling;
+ }
+
+ if (pstate_funcs.get_aperf_mperf_shift)
+ cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift();
+
+ if (pstate_funcs.get_vid)
+ pstate_funcs.get_vid(cpu);
+
+ intel_pstate_set_min_pstate(cpu);
+}
+
+/*
+ * Long hold time will keep high perf limits for long time,
+ * which negatively impacts perf/watt for some workloads,
+ * like specpower. 3ms is based on experiements on some
+ * workoads.
+ */
+static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC;
+
+static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)
+{
+ u64 hwp_req = READ_ONCE(cpu->hwp_req_cached);
+ u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
+ u32 max_limit = (hwp_req & 0xff00) >> 8;
+ u32 min_limit = (hwp_req & 0xff);
+ u32 boost_level1;
+
+ /*
+ * Cases to consider (User changes via sysfs or boot time):
+ * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
+ * No boost, return.
+ * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
+ * Should result in one level boost only for P0.
+ * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
+ * Should result in two level boost:
+ * (min + p1)/2 and P1.
+ * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
+ * Should result in three level boost:
+ * (min + p1)/2, P1 and P0.
+ */
+
+ /* If max and min are equal or already at max, nothing to boost */
+ if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit)
+ return;
+
+ if (!cpu->hwp_boost_min)
+ cpu->hwp_boost_min = min_limit;
+
+ /* level at half way mark between min and guranteed */
+ boost_level1 = (HWP_GUARANTEED_PERF(hwp_cap) + min_limit) >> 1;
+
+ if (cpu->hwp_boost_min < boost_level1)
+ cpu->hwp_boost_min = boost_level1;
+ else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(hwp_cap))
+ cpu->hwp_boost_min = HWP_GUARANTEED_PERF(hwp_cap);
+ else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(hwp_cap) &&
+ max_limit != HWP_GUARANTEED_PERF(hwp_cap))
+ cpu->hwp_boost_min = max_limit;
+ else
+ return;
+
+ hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min;
+ wrmsrl(MSR_HWP_REQUEST, hwp_req);
+ cpu->last_update = cpu->sample.time;
+}
+
+static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)
+{
+ if (cpu->hwp_boost_min) {
+ bool expired;
+
+ /* Check if we are idle for hold time to boost down */
+ expired = time_after64(cpu->sample.time, cpu->last_update +
+ hwp_boost_hold_time_ns);
+ if (expired) {
+ wrmsrl(MSR_HWP_REQUEST, cpu->hwp_req_cached);
+ cpu->hwp_boost_min = 0;
+ }
+ }
+ cpu->last_update = cpu->sample.time;
+}
+
+static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,
+ u64 time)
+{
+ cpu->sample.time = time;
+
+ if (cpu->sched_flags & SCHED_CPUFREQ_IOWAIT) {
+ bool do_io = false;
+
+ cpu->sched_flags = 0;
+ /*
+ * Set iowait_boost flag and update time. Since IO WAIT flag
+ * is set all the time, we can't just conclude that there is
+ * some IO bound activity is scheduled on this CPU with just
+ * one occurrence. If we receive at least two in two
+ * consecutive ticks, then we treat as boost candidate.
+ */
+ if (time_before64(time, cpu->last_io_update + 2 * TICK_NSEC))
+ do_io = true;
+
+ cpu->last_io_update = time;
+
+ if (do_io)
+ intel_pstate_hwp_boost_up(cpu);
+
+ } else {
+ intel_pstate_hwp_boost_down(cpu);
+ }
+}
+
+static inline void intel_pstate_update_util_hwp(struct update_util_data *data,
+ u64 time, unsigned int flags)
+{
+ struct cpudata *cpu = container_of(data, struct cpudata, update_util);
+
+ cpu->sched_flags |= flags;
+
+ if (smp_processor_id() == cpu->cpu)
+ intel_pstate_update_util_hwp_local(cpu, time);
+}
+
+static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
+{
+ struct sample *sample = &cpu->sample;
+
+ sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
+}
+
+static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
+{
+ u64 aperf, mperf;
+ unsigned long flags;
+ u64 tsc;
+
+ local_irq_save(flags);
+ rdmsrl(MSR_IA32_APERF, aperf);
+ rdmsrl(MSR_IA32_MPERF, mperf);
+ tsc = rdtsc();
+ if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
+ local_irq_restore(flags);
+ return false;
+ }
+ local_irq_restore(flags);
+
+ cpu->last_sample_time = cpu->sample.time;
+ cpu->sample.time = time;
+ cpu->sample.aperf = aperf;
+ cpu->sample.mperf = mperf;
+ cpu->sample.tsc = tsc;
+ cpu->sample.aperf -= cpu->prev_aperf;
+ cpu->sample.mperf -= cpu->prev_mperf;
+ cpu->sample.tsc -= cpu->prev_tsc;
+
+ cpu->prev_aperf = aperf;
+ cpu->prev_mperf = mperf;
+ cpu->prev_tsc = tsc;
+ /*
+ * First time this function is invoked in a given cycle, all of the
+ * previous sample data fields are equal to zero or stale and they must
+ * be populated with meaningful numbers for things to work, so assume
+ * that sample.time will always be reset before setting the utilization
+ * update hook and make the caller skip the sample then.
+ */
+ if (cpu->last_sample_time) {
+ intel_pstate_calc_avg_perf(cpu);
+ return true;
+ }
+ return false;
+}
+
+static inline int32_t get_avg_frequency(struct cpudata *cpu)
+{
+ return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz);
+}
+
+static inline int32_t get_avg_pstate(struct cpudata *cpu)
+{
+ return mul_ext_fp(cpu->pstate.max_pstate_physical,
+ cpu->sample.core_avg_perf);
+}
+
+static inline int32_t get_target_pstate(struct cpudata *cpu)
+{
+ struct sample *sample = &cpu->sample;
+ int32_t busy_frac;
+ int target, avg_pstate;
+
+ busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift,
+ sample->tsc);
+
+ if (busy_frac < cpu->iowait_boost)
+ busy_frac = cpu->iowait_boost;
+
+ sample->busy_scaled = busy_frac * 100;
+
+ target = global.no_turbo || global.turbo_disabled ?
+ cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
+ target += target >> 2;
+ target = mul_fp(target, busy_frac);
+ if (target < cpu->pstate.min_pstate)
+ target = cpu->pstate.min_pstate;
+
+ /*
+ * If the average P-state during the previous cycle was higher than the
+ * current target, add 50% of the difference to the target to reduce
+ * possible performance oscillations and offset possible performance
+ * loss related to moving the workload from one CPU to another within
+ * a package/module.
+ */
+ avg_pstate = get_avg_pstate(cpu);
+ if (avg_pstate > target)
+ target += (avg_pstate - target) >> 1;
+
+ return target;
+}
+
+static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
+{
+ int min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio);
+ int max_pstate = max(min_pstate, cpu->max_perf_ratio);
+
+ return clamp_t(int, pstate, min_pstate, max_pstate);
+}
+
+static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
+{
+ if (pstate == cpu->pstate.current_pstate)
+ return;
+
+ cpu->pstate.current_pstate = pstate;
+ wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
+}
+
+static void intel_pstate_adjust_pstate(struct cpudata *cpu)
+{
+ int from = cpu->pstate.current_pstate;
+ struct sample *sample;
+ int target_pstate;
+
+ update_turbo_state();
+
+ target_pstate = get_target_pstate(cpu);
+ target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
+ trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
+ intel_pstate_update_pstate(cpu, target_pstate);
+
+ sample = &cpu->sample;
+ trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
+ fp_toint(sample->busy_scaled),
+ from,
+ cpu->pstate.current_pstate,
+ sample->mperf,
+ sample->aperf,
+ sample->tsc,
+ get_avg_frequency(cpu),
+ fp_toint(cpu->iowait_boost * 100));
+}
+
+static void intel_pstate_update_util(struct update_util_data *data, u64 time,
+ unsigned int flags)
+{
+ struct cpudata *cpu = container_of(data, struct cpudata, update_util);
+ u64 delta_ns;
+
+ /* Don't allow remote callbacks */
+ if (smp_processor_id() != cpu->cpu)
+ return;
+
+ delta_ns = time - cpu->last_update;
+ if (flags & SCHED_CPUFREQ_IOWAIT) {
+ /* Start over if the CPU may have been idle. */
+ if (delta_ns > TICK_NSEC) {
+ cpu->iowait_boost = ONE_EIGHTH_FP;
+ } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) {
+ cpu->iowait_boost <<= 1;
+ if (cpu->iowait_boost > int_tofp(1))
+ cpu->iowait_boost = int_tofp(1);
+ } else {
+ cpu->iowait_boost = ONE_EIGHTH_FP;
+ }
+ } else if (cpu->iowait_boost) {
+ /* Clear iowait_boost if the CPU may have been idle. */
+ if (delta_ns > TICK_NSEC)
+ cpu->iowait_boost = 0;
+ else
+ cpu->iowait_boost >>= 1;
+ }
+ cpu->last_update = time;
+ delta_ns = time - cpu->sample.time;
+ if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL)
+ return;
+
+ if (intel_pstate_sample(cpu, time))
+ intel_pstate_adjust_pstate(cpu);
+}
+
+static struct pstate_funcs core_funcs = {
+ .get_max = core_get_max_pstate,
+ .get_max_physical = core_get_max_pstate_physical,
+ .get_min = core_get_min_pstate,
+ .get_turbo = core_get_turbo_pstate,
+ .get_scaling = core_get_scaling,
+ .get_val = core_get_val,
+};
+
+static const struct pstate_funcs silvermont_funcs = {
+ .get_max = atom_get_max_pstate,
+ .get_max_physical = atom_get_max_pstate,
+ .get_min = atom_get_min_pstate,
+ .get_turbo = atom_get_turbo_pstate,
+ .get_val = atom_get_val,
+ .get_scaling = silvermont_get_scaling,
+ .get_vid = atom_get_vid,
+};
+
+static const struct pstate_funcs airmont_funcs = {
+ .get_max = atom_get_max_pstate,
+ .get_max_physical = atom_get_max_pstate,
+ .get_min = atom_get_min_pstate,
+ .get_turbo = atom_get_turbo_pstate,
+ .get_val = atom_get_val,
+ .get_scaling = airmont_get_scaling,
+ .get_vid = atom_get_vid,
+};
+
+static const struct pstate_funcs knl_funcs = {
+ .get_max = core_get_max_pstate,
+ .get_max_physical = core_get_max_pstate_physical,
+ .get_min = core_get_min_pstate,
+ .get_turbo = knl_get_turbo_pstate,
+ .get_aperf_mperf_shift = knl_get_aperf_mperf_shift,
+ .get_scaling = core_get_scaling,
+ .get_val = core_get_val,
+};
+
+#define X86_MATCH(model, policy) \
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
+ X86_FEATURE_APERFMPERF, &policy)
+
+static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
+ X86_MATCH(SANDYBRIDGE, core_funcs),
+ X86_MATCH(SANDYBRIDGE_X, core_funcs),
+ X86_MATCH(ATOM_SILVERMONT, silvermont_funcs),
+ X86_MATCH(IVYBRIDGE, core_funcs),
+ X86_MATCH(HASWELL, core_funcs),
+ X86_MATCH(BROADWELL, core_funcs),
+ X86_MATCH(IVYBRIDGE_X, core_funcs),
+ X86_MATCH(HASWELL_X, core_funcs),
+ X86_MATCH(HASWELL_L, core_funcs),
+ X86_MATCH(HASWELL_G, core_funcs),
+ X86_MATCH(BROADWELL_G, core_funcs),
+ X86_MATCH(ATOM_AIRMONT, airmont_funcs),
+ X86_MATCH(SKYLAKE_L, core_funcs),
+ X86_MATCH(BROADWELL_X, core_funcs),
+ X86_MATCH(SKYLAKE, core_funcs),
+ X86_MATCH(BROADWELL_D, core_funcs),
+ X86_MATCH(XEON_PHI_KNL, knl_funcs),
+ X86_MATCH(XEON_PHI_KNM, knl_funcs),
+ X86_MATCH(ATOM_GOLDMONT, core_funcs),
+ X86_MATCH(ATOM_GOLDMONT_PLUS, core_funcs),
+ X86_MATCH(SKYLAKE_X, core_funcs),
+ X86_MATCH(COMETLAKE, core_funcs),
+ X86_MATCH(ICELAKE_X, core_funcs),
+ X86_MATCH(TIGERLAKE, core_funcs),
+ X86_MATCH(SAPPHIRERAPIDS_X, core_funcs),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
+
+static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
+ X86_MATCH(BROADWELL_D, core_funcs),
+ X86_MATCH(BROADWELL_X, core_funcs),
+ X86_MATCH(SKYLAKE_X, core_funcs),
+ X86_MATCH(ICELAKE_X, core_funcs),
+ X86_MATCH(SAPPHIRERAPIDS_X, core_funcs),
+ {}
+};
+
+static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
+ X86_MATCH(KABYLAKE, core_funcs),
+ {}
+};
+
+static int intel_pstate_init_cpu(unsigned int cpunum)
+{
+ struct cpudata *cpu;
+
+ cpu = all_cpu_data[cpunum];
+
+ if (!cpu) {
+ cpu = kzalloc(sizeof(*cpu), GFP_KERNEL);
+ if (!cpu)
+ return -ENOMEM;
+
+ WRITE_ONCE(all_cpu_data[cpunum], cpu);
+
+ cpu->cpu = cpunum;
+
+ cpu->epp_default = -EINVAL;
+
+ if (hwp_active) {
+ intel_pstate_hwp_enable(cpu);
+
+ if (intel_pstate_acpi_pm_profile_server())
+ hwp_boost = true;
+ }
+ } else if (hwp_active) {
+ /*
+ * Re-enable HWP in case this happens after a resume from ACPI
+ * S3 if the CPU was offline during the whole system/resume
+ * cycle.
+ */
+ intel_pstate_hwp_reenable(cpu);
+ }
+
+ cpu->epp_powersave = -EINVAL;
+ cpu->epp_policy = 0;
+
+ intel_pstate_get_cpu_pstates(cpu);
+
+ pr_debug("controlling: cpu %d\n", cpunum);
+
+ return 0;
+}
+
+static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
+{
+ struct cpudata *cpu = all_cpu_data[cpu_num];
+
+ if (hwp_active && !hwp_boost)
+ return;
+
+ if (cpu->update_util_set)
+ return;
+
+ /* Prevent intel_pstate_update_util() from using stale data. */
+ cpu->sample.time = 0;
+ cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
+ (hwp_active ?
+ intel_pstate_update_util_hwp :
+ intel_pstate_update_util));
+ cpu->update_util_set = true;
+}
+
+static void intel_pstate_clear_update_util_hook(unsigned int cpu)
+{
+ struct cpudata *cpu_data = all_cpu_data[cpu];
+
+ if (!cpu_data->update_util_set)
+ return;
+
+ cpufreq_remove_update_util_hook(cpu);
+ cpu_data->update_util_set = false;
+ synchronize_rcu();
+}
+
+static int intel_pstate_get_max_freq(struct cpudata *cpu)
+{
+ return global.turbo_disabled || global.no_turbo ?
+ cpu->pstate.max_freq : cpu->pstate.turbo_freq;
+}
+
+static void intel_pstate_update_perf_limits(struct cpudata *cpu,
+ unsigned int policy_min,
+ unsigned int policy_max)
+{
+ int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
+ int32_t max_policy_perf, min_policy_perf;
+
+ max_policy_perf = policy_max / perf_ctl_scaling;
+ if (policy_max == policy_min) {
+ min_policy_perf = max_policy_perf;
+ } else {
+ min_policy_perf = policy_min / perf_ctl_scaling;
+ min_policy_perf = clamp_t(int32_t, min_policy_perf,
+ 0, max_policy_perf);
+ }
+
+ /*
+ * HWP needs some special consideration, because HWP_REQUEST uses
+ * abstract values to represent performance rather than pure ratios.
+ */
+ if (hwp_active && cpu->pstate.scaling != perf_ctl_scaling) {
+ int freq;
+
+ freq = max_policy_perf * perf_ctl_scaling;
+ max_policy_perf = intel_pstate_freq_to_hwp(cpu, freq);
+ freq = min_policy_perf * perf_ctl_scaling;
+ min_policy_perf = intel_pstate_freq_to_hwp(cpu, freq);
+ }
+
+ pr_debug("cpu:%d min_policy_perf:%d max_policy_perf:%d\n",
+ cpu->cpu, min_policy_perf, max_policy_perf);
+
+ /* Normalize user input to [min_perf, max_perf] */
+ if (per_cpu_limits) {
+ cpu->min_perf_ratio = min_policy_perf;
+ cpu->max_perf_ratio = max_policy_perf;
+ } else {
+ int turbo_max = cpu->pstate.turbo_pstate;
+ int32_t global_min, global_max;
+
+ /* Global limits are in percent of the maximum turbo P-state. */
+ global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
+ global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
+ global_min = clamp_t(int32_t, global_min, 0, global_max);
+
+ pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu->cpu,
+ global_min, global_max);
+
+ cpu->min_perf_ratio = max(min_policy_perf, global_min);
+ cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf);
+ cpu->max_perf_ratio = min(max_policy_perf, global_max);
+ cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio);
+
+ /* Make sure min_perf <= max_perf */
+ cpu->min_perf_ratio = min(cpu->min_perf_ratio,
+ cpu->max_perf_ratio);
+
+ }
+ pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu->cpu,
+ cpu->max_perf_ratio,
+ cpu->min_perf_ratio);
+}
+
+static int intel_pstate_set_policy(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu;
+
+ if (!policy->cpuinfo.max_freq)
+ return -ENODEV;
+
+ pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
+ policy->cpuinfo.max_freq, policy->max);
+
+ cpu = all_cpu_data[policy->cpu];
+ cpu->policy = policy->policy;
+
+ mutex_lock(&intel_pstate_limits_lock);
+
+ intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
+
+ if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
+ /*
+ * NOHZ_FULL CPUs need this as the governor callback may not
+ * be invoked on them.
+ */
+ intel_pstate_clear_update_util_hook(policy->cpu);
+ intel_pstate_max_within_limits(cpu);
+ } else {
+ intel_pstate_set_update_util_hook(policy->cpu);
+ }
+
+ if (hwp_active) {
+ /*
+ * When hwp_boost was active before and dynamically it
+ * was turned off, in that case we need to clear the
+ * update util hook.
+ */
+ if (!hwp_boost)
+ intel_pstate_clear_update_util_hook(policy->cpu);
+ intel_pstate_hwp_set(policy->cpu);
+ }
+ /*
+ * policy->cur is never updated with the intel_pstate driver, but it
+ * is used as a stale frequency value. So, keep it within limits.
+ */
+ policy->cur = policy->min;
+
+ mutex_unlock(&intel_pstate_limits_lock);
+
+ return 0;
+}
+
+static void intel_pstate_adjust_policy_max(struct cpudata *cpu,
+ struct cpufreq_policy_data *policy)
+{
+ if (!hwp_active &&
+ cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
+ policy->max < policy->cpuinfo.max_freq &&
+ policy->max > cpu->pstate.max_freq) {
+ pr_debug("policy->max > max non turbo frequency\n");
+ policy->max = policy->cpuinfo.max_freq;
+ }
+}
+
+static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,
+ struct cpufreq_policy_data *policy)
+{
+ int max_freq;
+
+ update_turbo_state();
+ if (hwp_active) {
+ intel_pstate_get_hwp_cap(cpu);
+ max_freq = global.no_turbo || global.turbo_disabled ?
+ cpu->pstate.max_freq : cpu->pstate.turbo_freq;
+ } else {
+ max_freq = intel_pstate_get_max_freq(cpu);
+ }
+ cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq);
+
+ intel_pstate_adjust_policy_max(cpu, policy);
+}
+
+static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
+{
+ intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy);
+
+ return 0;
+}
+
+static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ pr_debug("CPU %d going offline\n", cpu->cpu);
+
+ if (cpu->suspended)
+ return 0;
+
+ /*
+ * If the CPU is an SMT thread and it goes offline with the performance
+ * settings different from the minimum, it will prevent its sibling
+ * from getting to lower performance levels, so force the minimum
+ * performance on CPU offline to prevent that from happening.
+ */
+ if (hwp_active)
+ intel_pstate_hwp_offline(cpu);
+ else
+ intel_pstate_set_min_pstate(cpu);
+
+ intel_pstate_exit_perf_limits(policy);
+
+ return 0;
+}
+
+static int intel_pstate_cpu_online(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ pr_debug("CPU %d going online\n", cpu->cpu);
+
+ intel_pstate_init_acpi_perf_limits(policy);
+
+ if (hwp_active) {
+ /*
+ * Re-enable HWP and clear the "suspended" flag to let "resume"
+ * know that it need not do that.
+ */
+ intel_pstate_hwp_reenable(cpu);
+ cpu->suspended = false;
+ }
+
+ return 0;
+}
+
+static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
+{
+ intel_pstate_clear_update_util_hook(policy->cpu);
+
+ return intel_cpufreq_cpu_offline(policy);
+}
+
+static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
+{
+ pr_debug("CPU %d exiting\n", policy->cpu);
+
+ policy->fast_switch_possible = false;
+
+ return 0;
+}
+
+static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpudata *cpu;
+ int rc;
+
+ rc = intel_pstate_init_cpu(policy->cpu);
+ if (rc)
+ return rc;
+
+ cpu = all_cpu_data[policy->cpu];
+
+ cpu->max_perf_ratio = 0xFF;
+ cpu->min_perf_ratio = 0;
+
+ /* cpuinfo and default policy values */
+ policy->cpuinfo.min_freq = cpu->pstate.min_freq;
+ update_turbo_state();
+ global.turbo_disabled_mf = global.turbo_disabled;
+ policy->cpuinfo.max_freq = global.turbo_disabled ?
+ cpu->pstate.max_freq : cpu->pstate.turbo_freq;
+
+ policy->min = policy->cpuinfo.min_freq;
+ policy->max = policy->cpuinfo.max_freq;
+
+ intel_pstate_init_acpi_perf_limits(policy);
+
+ policy->fast_switch_possible = true;
+
+ return 0;
+}
+
+static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
+{
+ int ret = __intel_pstate_cpu_init(policy);
+
+ if (ret)
+ return ret;
+
+ /*
+ * Set the policy to powersave to provide a valid fallback value in case
+ * the default cpufreq governor is neither powersave nor performance.
+ */
+ policy->policy = CPUFREQ_POLICY_POWERSAVE;
+
+ if (hwp_active) {
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ cpu->epp_cached = intel_pstate_get_epp(cpu, 0);
+ }
+
+ return 0;
+}
+
+static struct cpufreq_driver intel_pstate = {
+ .flags = CPUFREQ_CONST_LOOPS,
+ .verify = intel_pstate_verify_policy,
+ .setpolicy = intel_pstate_set_policy,
+ .suspend = intel_pstate_suspend,
+ .resume = intel_pstate_resume,
+ .init = intel_pstate_cpu_init,
+ .exit = intel_pstate_cpu_exit,
+ .offline = intel_pstate_cpu_offline,
+ .online = intel_pstate_cpu_online,
+ .update_limits = intel_pstate_update_limits,
+ .name = "intel_pstate",
+};
+
+static int intel_cpufreq_verify_policy(struct cpufreq_policy_data *policy)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+
+ intel_pstate_verify_cpu_policy(cpu, policy);
+ intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
+
+ return 0;
+}
+
+/* Use of trace in passive mode:
+ *
+ * In passive mode the trace core_busy field (also known as the
+ * performance field, and lablelled as such on the graphs; also known as
+ * core_avg_perf) is not needed and so is re-assigned to indicate if the
+ * driver call was via the normal or fast switch path. Various graphs
+ * output from the intel_pstate_tracer.py utility that include core_busy
+ * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
+ * so we use 10 to indicate the normal path through the driver, and
+ * 90 to indicate the fast switch path through the driver.
+ * The scaled_busy field is not used, and is set to 0.
+ */
+
+#define INTEL_PSTATE_TRACE_TARGET 10
+#define INTEL_PSTATE_TRACE_FAST_SWITCH 90
+
+static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)
+{
+ struct sample *sample;
+
+ if (!trace_pstate_sample_enabled())
+ return;
+
+ if (!intel_pstate_sample(cpu, ktime_get()))
+ return;
+
+ sample = &cpu->sample;
+ trace_pstate_sample(trace_type,
+ 0,
+ old_pstate,
+ cpu->pstate.current_pstate,
+ sample->mperf,
+ sample->aperf,
+ sample->tsc,
+ get_avg_frequency(cpu),
+ fp_toint(cpu->iowait_boost * 100));
+}
+
+static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max,
+ u32 desired, bool fast_switch)
+{
+ u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev;
+
+ value &= ~HWP_MIN_PERF(~0L);
+ value |= HWP_MIN_PERF(min);
+
+ value &= ~HWP_MAX_PERF(~0L);
+ value |= HWP_MAX_PERF(max);
+
+ value &= ~HWP_DESIRED_PERF(~0L);
+ value |= HWP_DESIRED_PERF(desired);
+
+ if (value == prev)
+ return;
+
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+ if (fast_switch)
+ wrmsrl(MSR_HWP_REQUEST, value);
+ else
+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
+}
+
+static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu,
+ u32 target_pstate, bool fast_switch)
+{
+ if (fast_switch)
+ wrmsrl(MSR_IA32_PERF_CTL,
+ pstate_funcs.get_val(cpu, target_pstate));
+ else
+ wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
+ pstate_funcs.get_val(cpu, target_pstate));
+}
+
+static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy,
+ int target_pstate, bool fast_switch)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ int old_pstate = cpu->pstate.current_pstate;
+
+ target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
+ if (hwp_active) {
+ int max_pstate = policy->strict_target ?
+ target_pstate : cpu->max_perf_ratio;
+
+ intel_cpufreq_hwp_update(cpu, target_pstate, max_pstate, 0,
+ fast_switch);
+ } else if (target_pstate != old_pstate) {
+ intel_cpufreq_perf_ctl_update(cpu, target_pstate, fast_switch);
+ }
+
+ cpu->pstate.current_pstate = target_pstate;
+
+ intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH :
+ INTEL_PSTATE_TRACE_TARGET, old_pstate);
+
+ return target_pstate;
+}
+
+static int intel_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ struct cpufreq_freqs freqs;
+ int target_pstate;
+
+ update_turbo_state();
+
+ freqs.old = policy->cur;
+ freqs.new = target_freq;
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+
+ target_pstate = intel_pstate_freq_to_hwp_rel(cpu, freqs.new, relation);
+ target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false);
+
+ freqs.new = target_pstate * cpu->pstate.scaling;
+
+ cpufreq_freq_transition_end(policy, &freqs, false);
+
+ return 0;
+}
+
+static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ int target_pstate;
+
+ update_turbo_state();
+
+ target_pstate = intel_pstate_freq_to_hwp(cpu, target_freq);
+
+ target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true);
+
+ return target_pstate * cpu->pstate.scaling;
+}
+
+static void intel_cpufreq_adjust_perf(unsigned int cpunum,
+ unsigned long min_perf,
+ unsigned long target_perf,
+ unsigned long capacity)
+{
+ struct cpudata *cpu = all_cpu_data[cpunum];
+ u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
+ int old_pstate = cpu->pstate.current_pstate;
+ int cap_pstate, min_pstate, max_pstate, target_pstate;
+
+ update_turbo_state();
+ cap_pstate = global.turbo_disabled ? HWP_GUARANTEED_PERF(hwp_cap) :
+ HWP_HIGHEST_PERF(hwp_cap);
+
+ /* Optimization: Avoid unnecessary divisions. */
+
+ target_pstate = cap_pstate;
+ if (target_perf < capacity)
+ target_pstate = DIV_ROUND_UP(cap_pstate * target_perf, capacity);
+
+ min_pstate = cap_pstate;
+ if (min_perf < capacity)
+ min_pstate = DIV_ROUND_UP(cap_pstate * min_perf, capacity);
+
+ if (min_pstate < cpu->pstate.min_pstate)
+ min_pstate = cpu->pstate.min_pstate;
+
+ if (min_pstate < cpu->min_perf_ratio)
+ min_pstate = cpu->min_perf_ratio;
+
+ max_pstate = min(cap_pstate, cpu->max_perf_ratio);
+ if (max_pstate < min_pstate)
+ max_pstate = min_pstate;
+
+ target_pstate = clamp_t(int, target_pstate, min_pstate, max_pstate);
+
+ intel_cpufreq_hwp_update(cpu, min_pstate, max_pstate, target_pstate, true);
+
+ cpu->pstate.current_pstate = target_pstate;
+ intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate);
+}
+
+static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ struct freq_qos_request *req;
+ struct cpudata *cpu;
+ struct device *dev;
+ int ret, freq;
+
+ dev = get_cpu_device(policy->cpu);
+ if (!dev)
+ return -ENODEV;
+
+ ret = __intel_pstate_cpu_init(policy);
+ if (ret)
+ return ret;
+
+ policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
+ /* This reflects the intel_pstate_get_cpu_pstates() setting. */
+ policy->cur = policy->cpuinfo.min_freq;
+
+ req = kcalloc(2, sizeof(*req), GFP_KERNEL);
+ if (!req) {
+ ret = -ENOMEM;
+ goto pstate_exit;
+ }
+
+ cpu = all_cpu_data[policy->cpu];
+
+ if (hwp_active) {
+ u64 value;
+
+ policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP;
+
+ intel_pstate_get_hwp_cap(cpu);
+
+ rdmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value);
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+
+ cpu->epp_cached = intel_pstate_get_epp(cpu, value);
+ } else {
+ policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
+ }
+
+ freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100);
+
+ ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN,
+ freq);
+ if (ret < 0) {
+ dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
+ goto free_req;
+ }
+
+ freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.max_perf_pct, 100);
+
+ ret = freq_qos_add_request(&policy->constraints, req + 1, FREQ_QOS_MAX,
+ freq);
+ if (ret < 0) {
+ dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
+ goto remove_min_req;
+ }
+
+ policy->driver_data = req;
+
+ return 0;
+
+remove_min_req:
+ freq_qos_remove_request(req);
+free_req:
+ kfree(req);
+pstate_exit:
+ intel_pstate_exit_perf_limits(policy);
+
+ return ret;
+}
+
+static int intel_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct freq_qos_request *req;
+
+ req = policy->driver_data;
+
+ freq_qos_remove_request(req + 1);
+ freq_qos_remove_request(req);
+ kfree(req);
+
+ return intel_pstate_cpu_exit(policy);
+}
+
+static int intel_cpufreq_suspend(struct cpufreq_policy *policy)
+{
+ intel_pstate_suspend(policy);
+
+ if (hwp_active) {
+ struct cpudata *cpu = all_cpu_data[policy->cpu];
+ u64 value = READ_ONCE(cpu->hwp_req_cached);
+
+ /*
+ * Clear the desired perf field in MSR_HWP_REQUEST in case
+ * intel_cpufreq_adjust_perf() is in use and the last value
+ * written by it may not be suitable.
+ */
+ value &= ~HWP_DESIRED_PERF(~0L);
+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
+ WRITE_ONCE(cpu->hwp_req_cached, value);
+ }
+
+ return 0;
+}
+
+static struct cpufreq_driver intel_cpufreq = {
+ .flags = CPUFREQ_CONST_LOOPS,
+ .verify = intel_cpufreq_verify_policy,
+ .target = intel_cpufreq_target,
+ .fast_switch = intel_cpufreq_fast_switch,
+ .init = intel_cpufreq_cpu_init,
+ .exit = intel_cpufreq_cpu_exit,
+ .offline = intel_cpufreq_cpu_offline,
+ .online = intel_pstate_cpu_online,
+ .suspend = intel_cpufreq_suspend,
+ .resume = intel_pstate_resume,
+ .update_limits = intel_pstate_update_limits,
+ .name = "intel_cpufreq",
+};
+
+static struct cpufreq_driver *default_driver;
+
+static void intel_pstate_driver_cleanup(void)
+{
+ unsigned int cpu;
+
+ cpus_read_lock();
+ for_each_online_cpu(cpu) {
+ if (all_cpu_data[cpu]) {
+ if (intel_pstate_driver == &intel_pstate)
+ intel_pstate_clear_update_util_hook(cpu);
+
+ spin_lock(&hwp_notify_lock);
+ kfree(all_cpu_data[cpu]);
+ WRITE_ONCE(all_cpu_data[cpu], NULL);
+ spin_unlock(&hwp_notify_lock);
+ }
+ }
+ cpus_read_unlock();
+
+ intel_pstate_driver = NULL;
+}
+
+static int intel_pstate_register_driver(struct cpufreq_driver *driver)
+{
+ int ret;
+
+ if (driver == &intel_pstate)
+ intel_pstate_sysfs_expose_hwp_dynamic_boost();
+
+ memset(&global, 0, sizeof(global));
+ global.max_perf_pct = 100;
+
+ intel_pstate_driver = driver;
+ ret = cpufreq_register_driver(intel_pstate_driver);
+ if (ret) {
+ intel_pstate_driver_cleanup();
+ return ret;
+ }
+
+ global.min_perf_pct = min_perf_pct_min();
+
+ return 0;
+}
+
+static ssize_t intel_pstate_show_status(char *buf)
+{
+ if (!intel_pstate_driver)
+ return sprintf(buf, "off\n");
+
+ return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
+ "active" : "passive");
+}
+
+static int intel_pstate_update_status(const char *buf, size_t size)
+{
+ if (size == 3 && !strncmp(buf, "off", size)) {
+ if (!intel_pstate_driver)
+ return -EINVAL;
+
+ if (hwp_active)
+ return -EBUSY;
+
+ cpufreq_unregister_driver(intel_pstate_driver);
+ intel_pstate_driver_cleanup();
+ return 0;
+ }
+
+ if (size == 6 && !strncmp(buf, "active", size)) {
+ if (intel_pstate_driver) {
+ if (intel_pstate_driver == &intel_pstate)
+ return 0;
+
+ cpufreq_unregister_driver(intel_pstate_driver);
+ }
+
+ return intel_pstate_register_driver(&intel_pstate);
+ }
+
+ if (size == 7 && !strncmp(buf, "passive", size)) {
+ if (intel_pstate_driver) {
+ if (intel_pstate_driver == &intel_cpufreq)
+ return 0;
+
+ cpufreq_unregister_driver(intel_pstate_driver);
+ intel_pstate_sysfs_hide_hwp_dynamic_boost();
+ }
+
+ return intel_pstate_register_driver(&intel_cpufreq);
+ }
+
+ return -EINVAL;
+}
+
+static int no_load __initdata;
+static int no_hwp __initdata;
+static int hwp_only __initdata;
+static unsigned int force_load __initdata;
+
+static int __init intel_pstate_msrs_not_valid(void)
+{
+ if (!pstate_funcs.get_max(0) ||
+ !pstate_funcs.get_min(0) ||
+ !pstate_funcs.get_turbo(0))
+ return -ENODEV;
+
+ return 0;
+}
+
+static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
+{
+ pstate_funcs.get_max = funcs->get_max;
+ pstate_funcs.get_max_physical = funcs->get_max_physical;
+ pstate_funcs.get_min = funcs->get_min;
+ pstate_funcs.get_turbo = funcs->get_turbo;
+ pstate_funcs.get_scaling = funcs->get_scaling;
+ pstate_funcs.get_val = funcs->get_val;
+ pstate_funcs.get_vid = funcs->get_vid;
+ pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift;
+}
+
+#ifdef CONFIG_ACPI
+
+static bool __init intel_pstate_no_acpi_pss(void)
+{
+ int i;
+
+ for_each_possible_cpu(i) {
+ acpi_status status;
+ union acpi_object *pss;
+ struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+ struct acpi_processor *pr = per_cpu(processors, i);
+
+ if (!pr)
+ continue;
+
+ status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
+ if (ACPI_FAILURE(status))
+ continue;
+
+ pss = buffer.pointer;
+ if (pss && pss->type == ACPI_TYPE_PACKAGE) {
+ kfree(pss);
+ return false;
+ }
+
+ kfree(pss);
+ }
+
+ pr_debug("ACPI _PSS not found\n");
+ return true;
+}
+
+static bool __init intel_pstate_no_acpi_pcch(void)
+{
+ acpi_status status;
+ acpi_handle handle;
+
+ status = acpi_get_handle(NULL, "\\_SB", &handle);
+ if (ACPI_FAILURE(status))
+ goto not_found;
+
+ if (acpi_has_method(handle, "PCCH"))
+ return false;
+
+not_found:
+ pr_debug("ACPI PCCH not found\n");
+ return true;
+}
+
+static bool __init intel_pstate_has_acpi_ppc(void)
+{
+ int i;
+
+ for_each_possible_cpu(i) {
+ struct acpi_processor *pr = per_cpu(processors, i);
+
+ if (!pr)
+ continue;
+ if (acpi_has_method(pr->handle, "_PPC"))
+ return true;
+ }
+ pr_debug("ACPI _PPC not found\n");
+ return false;
+}
+
+enum {
+ PSS,
+ PPC,
+};
+
+/* Hardware vendor-specific info that has its own power management modes */
+static struct acpi_platform_list plat_info[] __initdata = {
+ {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS},
+ {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
+ { } /* End */
+};
+
+#define BITMASK_OOB (BIT(8) | BIT(18))
+
+static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
+{
+ const struct x86_cpu_id *id;
+ u64 misc_pwr;
+ int idx;
+
+ id = x86_match_cpu(intel_pstate_cpu_oob_ids);
+ if (id) {
+ rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
+ if (misc_pwr & BITMASK_OOB) {
+ pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
+ pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
+ return true;
+ }
+ }
+
+ idx = acpi_match_platform_list(plat_info);
+ if (idx < 0)
+ return false;
+
+ switch (plat_info[idx].data) {
+ case PSS:
+ if (!intel_pstate_no_acpi_pss())
+ return false;
+
+ return intel_pstate_no_acpi_pcch();
+ case PPC:
+ return intel_pstate_has_acpi_ppc() && !force_load;
+ }
+
+ return false;
+}
+
+static void intel_pstate_request_control_from_smm(void)
+{
+ /*
+ * It may be unsafe to request P-states control from SMM if _PPC support
+ * has not been enabled.
+ */
+ if (acpi_ppc)
+ acpi_processor_pstate_control();
+}
+#else /* CONFIG_ACPI not enabled */
+static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
+static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
+static inline void intel_pstate_request_control_from_smm(void) {}
+#endif /* CONFIG_ACPI */
+
+#define INTEL_PSTATE_HWP_BROADWELL 0x01
+
+#define X86_MATCH_HWP(model, hwp_mode) \
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
+ X86_FEATURE_HWP, hwp_mode)
+
+static const struct x86_cpu_id hwp_support_ids[] __initconst = {
+ X86_MATCH_HWP(BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
+ X86_MATCH_HWP(BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL),
+ X86_MATCH_HWP(ANY, 0),
+ {}
+};
+
+static bool intel_pstate_hwp_is_enabled(void)
+{
+ u64 value;
+
+ rdmsrl(MSR_PM_ENABLE, value);
+ return !!(value & 0x1);
+}
+
+static const struct x86_cpu_id intel_epp_balance_perf[] = {
+ /*
+ * Set EPP value as 102, this is the max suggested EPP
+ * which can result in one core turbo frequency for
+ * AlderLake Mobile CPUs.
+ */
+ X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, 102),
+ X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, 32),
+ {}
+};
+
+static int __init intel_pstate_init(void)
+{
+ static struct cpudata **_all_cpu_data;
+ const struct x86_cpu_id *id;
+ int rc;
+
+ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+ return -ENODEV;
+
+ id = x86_match_cpu(hwp_support_ids);
+ if (id) {
+ hwp_forced = intel_pstate_hwp_is_enabled();
+
+ if (hwp_forced)
+ pr_info("HWP enabled by BIOS\n");
+ else if (no_load)
+ return -ENODEV;
+
+ copy_cpu_funcs(&core_funcs);
+ /*
+ * Avoid enabling HWP for processors without EPP support,
+ * because that means incomplete HWP implementation which is a
+ * corner case and supporting it is generally problematic.
+ *
+ * If HWP is enabled already, though, there is no choice but to
+ * deal with it.
+ */
+ if ((!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) || hwp_forced) {
+ WRITE_ONCE(hwp_active, 1);
+ hwp_mode_bdw = id->driver_data;
+ intel_pstate.attr = hwp_cpufreq_attrs;
+ intel_cpufreq.attr = hwp_cpufreq_attrs;
+ intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS;
+ intel_cpufreq.adjust_perf = intel_cpufreq_adjust_perf;
+ if (!default_driver)
+ default_driver = &intel_pstate;
+
+ pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling;
+
+ goto hwp_cpu_matched;
+ }
+ pr_info("HWP not enabled\n");
+ } else {
+ if (no_load)
+ return -ENODEV;
+
+ id = x86_match_cpu(intel_pstate_cpu_ids);
+ if (!id) {
+ pr_info("CPU model not supported\n");
+ return -ENODEV;
+ }
+
+ copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
+ }
+
+ if (intel_pstate_msrs_not_valid()) {
+ pr_info("Invalid MSRs\n");
+ return -ENODEV;
+ }
+ /* Without HWP start in the passive mode. */
+ if (!default_driver)
+ default_driver = &intel_cpufreq;
+
+hwp_cpu_matched:
+ /*
+ * The Intel pstate driver will be ignored if the platform
+ * firmware has its own power management modes.
+ */
+ if (intel_pstate_platform_pwr_mgmt_exists()) {
+ pr_info("P-states controlled by the platform\n");
+ return -ENODEV;
+ }
+
+ if (!hwp_active && hwp_only)
+ return -ENOTSUPP;
+
+ pr_info("Intel P-state driver initializing\n");
+
+ _all_cpu_data = vzalloc(array_size(sizeof(void *), num_possible_cpus()));
+ if (!_all_cpu_data)
+ return -ENOMEM;
+
+ WRITE_ONCE(all_cpu_data, _all_cpu_data);
+
+ intel_pstate_request_control_from_smm();
+
+ intel_pstate_sysfs_expose_params();
+
+ if (hwp_active) {
+ const struct x86_cpu_id *id = x86_match_cpu(intel_epp_balance_perf);
+
+ if (id)
+ epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = id->driver_data;
+ }
+
+ mutex_lock(&intel_pstate_driver_lock);
+ rc = intel_pstate_register_driver(default_driver);
+ mutex_unlock(&intel_pstate_driver_lock);
+ if (rc) {
+ intel_pstate_sysfs_remove();
+ return rc;
+ }
+
+ if (hwp_active) {
+ const struct x86_cpu_id *id;
+
+ id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
+ if (id) {
+ set_power_ctl_ee_state(false);
+ pr_info("Disabling energy efficiency optimization\n");
+ }
+
+ pr_info("HWP enabled\n");
+ } else if (boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
+ pr_warn("Problematic setup: Hybrid processor with disabled HWP\n");
+ }
+
+ return 0;
+}
+device_initcall(intel_pstate_init);
+
+static int __init intel_pstate_setup(char *str)
+{
+ if (!str)
+ return -EINVAL;
+
+ if (!strcmp(str, "disable"))
+ no_load = 1;
+ else if (!strcmp(str, "active"))
+ default_driver = &intel_pstate;
+ else if (!strcmp(str, "passive"))
+ default_driver = &intel_cpufreq;
+
+ if (!strcmp(str, "no_hwp"))
+ no_hwp = 1;
+
+ if (!strcmp(str, "force"))
+ force_load = 1;
+ if (!strcmp(str, "hwp_only"))
+ hwp_only = 1;
+ if (!strcmp(str, "per_cpu_perf_limits"))
+ per_cpu_limits = true;
+
+#ifdef CONFIG_ACPI
+ if (!strcmp(str, "support_acpi_ppc"))
+ acpi_ppc = true;
+#endif
+
+ return 0;
+}
+early_param("intel_pstate", intel_pstate_setup);
+
+MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
+MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c
new file mode 100644
index 0000000000..fd20b986d1
--- /dev/null
+++ b/drivers/cpufreq/kirkwood-cpufreq.c
@@ -0,0 +1,203 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * kirkwood_freq.c: cpufreq driver for the Marvell kirkwood
+ *
+ * Copyright (C) 2013 Andrew Lunn <andrew@lunn.ch>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/clk.h>
+#include <linux/cpufreq.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <asm/proc-fns.h>
+
+#define CPU_SW_INT_BLK BIT(28)
+
+static struct priv
+{
+ struct clk *cpu_clk;
+ struct clk *ddr_clk;
+ struct clk *powersave_clk;
+ struct device *dev;
+ void __iomem *base;
+} priv;
+
+#define STATE_CPU_FREQ 0x01
+#define STATE_DDR_FREQ 0x02
+
+/*
+ * Kirkwood can swap the clock to the CPU between two clocks:
+ *
+ * - cpu clk
+ * - ddr clk
+ *
+ * The frequencies are set at runtime before registering this table.
+ */
+static struct cpufreq_frequency_table kirkwood_freq_table[] = {
+ {0, STATE_CPU_FREQ, 0}, /* CPU uses cpuclk */
+ {0, STATE_DDR_FREQ, 0}, /* CPU uses ddrclk */
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+static unsigned int kirkwood_cpufreq_get_cpu_frequency(unsigned int cpu)
+{
+ return clk_get_rate(priv.powersave_clk) / 1000;
+}
+
+static int kirkwood_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ unsigned int state = kirkwood_freq_table[index].driver_data;
+ unsigned long reg;
+
+ local_irq_disable();
+
+ /* Disable interrupts to the CPU */
+ reg = readl_relaxed(priv.base);
+ reg |= CPU_SW_INT_BLK;
+ writel_relaxed(reg, priv.base);
+
+ switch (state) {
+ case STATE_CPU_FREQ:
+ clk_set_parent(priv.powersave_clk, priv.cpu_clk);
+ break;
+ case STATE_DDR_FREQ:
+ clk_set_parent(priv.powersave_clk, priv.ddr_clk);
+ break;
+ }
+
+ /* Wait-for-Interrupt, while the hardware changes frequency */
+ cpu_do_idle();
+
+ /* Enable interrupts to the CPU */
+ reg = readl_relaxed(priv.base);
+ reg &= ~CPU_SW_INT_BLK;
+ writel_relaxed(reg, priv.base);
+
+ local_irq_enable();
+
+ return 0;
+}
+
+/* Module init and exit code */
+static int kirkwood_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ cpufreq_generic_init(policy, kirkwood_freq_table, 5000);
+ return 0;
+}
+
+static struct cpufreq_driver kirkwood_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .get = kirkwood_cpufreq_get_cpu_frequency,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = kirkwood_cpufreq_target,
+ .init = kirkwood_cpufreq_cpu_init,
+ .name = "kirkwood-cpufreq",
+ .attr = cpufreq_generic_attr,
+};
+
+static int kirkwood_cpufreq_probe(struct platform_device *pdev)
+{
+ struct device_node *np;
+ int err;
+
+ priv.dev = &pdev->dev;
+
+ priv.base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(priv.base))
+ return PTR_ERR(priv.base);
+
+ np = of_cpu_device_node_get(0);
+ if (!np) {
+ dev_err(&pdev->dev, "failed to get cpu device node\n");
+ return -ENODEV;
+ }
+
+ priv.cpu_clk = of_clk_get_by_name(np, "cpu_clk");
+ if (IS_ERR(priv.cpu_clk)) {
+ dev_err(priv.dev, "Unable to get cpuclk\n");
+ err = PTR_ERR(priv.cpu_clk);
+ goto out_node;
+ }
+
+ err = clk_prepare_enable(priv.cpu_clk);
+ if (err) {
+ dev_err(priv.dev, "Unable to prepare cpuclk\n");
+ goto out_node;
+ }
+
+ kirkwood_freq_table[0].frequency = clk_get_rate(priv.cpu_clk) / 1000;
+
+ priv.ddr_clk = of_clk_get_by_name(np, "ddrclk");
+ if (IS_ERR(priv.ddr_clk)) {
+ dev_err(priv.dev, "Unable to get ddrclk\n");
+ err = PTR_ERR(priv.ddr_clk);
+ goto out_cpu;
+ }
+
+ err = clk_prepare_enable(priv.ddr_clk);
+ if (err) {
+ dev_err(priv.dev, "Unable to prepare ddrclk\n");
+ goto out_cpu;
+ }
+ kirkwood_freq_table[1].frequency = clk_get_rate(priv.ddr_clk) / 1000;
+
+ priv.powersave_clk = of_clk_get_by_name(np, "powersave");
+ if (IS_ERR(priv.powersave_clk)) {
+ dev_err(priv.dev, "Unable to get powersave\n");
+ err = PTR_ERR(priv.powersave_clk);
+ goto out_ddr;
+ }
+ err = clk_prepare_enable(priv.powersave_clk);
+ if (err) {
+ dev_err(priv.dev, "Unable to prepare powersave clk\n");
+ goto out_ddr;
+ }
+
+ err = cpufreq_register_driver(&kirkwood_cpufreq_driver);
+ if (err) {
+ dev_err(priv.dev, "Failed to register cpufreq driver\n");
+ goto out_powersave;
+ }
+
+ of_node_put(np);
+ return 0;
+
+out_powersave:
+ clk_disable_unprepare(priv.powersave_clk);
+out_ddr:
+ clk_disable_unprepare(priv.ddr_clk);
+out_cpu:
+ clk_disable_unprepare(priv.cpu_clk);
+out_node:
+ of_node_put(np);
+
+ return err;
+}
+
+static void kirkwood_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&kirkwood_cpufreq_driver);
+
+ clk_disable_unprepare(priv.powersave_clk);
+ clk_disable_unprepare(priv.ddr_clk);
+ clk_disable_unprepare(priv.cpu_clk);
+}
+
+static struct platform_driver kirkwood_cpufreq_platform_driver = {
+ .probe = kirkwood_cpufreq_probe,
+ .remove_new = kirkwood_cpufreq_remove,
+ .driver = {
+ .name = "kirkwood-cpufreq",
+ },
+};
+
+module_platform_driver(kirkwood_cpufreq_platform_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Andrew Lunn <andrew@lunn.ch");
+MODULE_DESCRIPTION("cpufreq driver for Marvell's kirkwood CPU");
+MODULE_ALIAS("platform:kirkwood-cpufreq");
diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c
new file mode 100644
index 0000000000..4c57c6725c
--- /dev/null
+++ b/drivers/cpufreq/longhaul.c
@@ -0,0 +1,1000 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * (C) 2001-2004 Dave Jones.
+ * (C) 2002 Padraig Brady. <padraig@antefacto.com>
+ *
+ * Based upon datasheets & sample CPUs kindly provided by VIA.
+ *
+ * VIA have currently 3 different versions of Longhaul.
+ * Version 1 (Longhaul) uses the BCR2 MSR at 0x1147.
+ * It is present only in Samuel 1 (C5A), Samuel 2 (C5B) stepping 0.
+ * Version 2 of longhaul is backward compatible with v1, but adds
+ * LONGHAUL MSR for purpose of both frequency and voltage scaling.
+ * Present in Samuel 2 (steppings 1-7 only) (C5B), and Ezra (C5C).
+ * Version 3 of longhaul got renamed to Powersaver and redesigned
+ * to use only the POWERSAVER MSR at 0x110a.
+ * It is present in Ezra-T (C5M), Nehemiah (C5X) and above.
+ * It's pretty much the same feature wise to longhaul v2, though
+ * there is provision for scaling FSB too, but this doesn't work
+ * too well in practice so we don't even try to use this.
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/delay.h>
+#include <linux/timex.h>
+#include <linux/io.h>
+#include <linux/acpi.h>
+
+#include <asm/msr.h>
+#include <asm/cpu_device_id.h>
+#include <acpi/processor.h>
+
+#include "longhaul.h"
+
+#define TYPE_LONGHAUL_V1 1
+#define TYPE_LONGHAUL_V2 2
+#define TYPE_POWERSAVER 3
+
+#define CPU_SAMUEL 1
+#define CPU_SAMUEL2 2
+#define CPU_EZRA 3
+#define CPU_EZRA_T 4
+#define CPU_NEHEMIAH 5
+#define CPU_NEHEMIAH_C 6
+
+/* Flags */
+#define USE_ACPI_C3 (1 << 1)
+#define USE_NORTHBRIDGE (1 << 2)
+
+static int cpu_model;
+static unsigned int numscales = 16;
+static unsigned int fsb;
+
+static const struct mV_pos *vrm_mV_table;
+static const unsigned char *mV_vrm_table;
+
+static unsigned int highest_speed, lowest_speed; /* kHz */
+static unsigned int minmult, maxmult;
+static int can_scale_voltage;
+static struct acpi_processor *pr;
+static struct acpi_processor_cx *cx;
+static u32 acpi_regs_addr;
+static u8 longhaul_flags;
+static unsigned int longhaul_index;
+
+/* Module parameters */
+static int scale_voltage;
+static int disable_acpi_c3;
+static int revid_errata;
+static int enable;
+
+/* Clock ratios multiplied by 10 */
+static int mults[32];
+static int eblcr[32];
+static int longhaul_version;
+static struct cpufreq_frequency_table *longhaul_table;
+
+static char speedbuffer[8];
+
+static char *print_speed(int speed)
+{
+ if (speed < 1000) {
+ snprintf(speedbuffer, sizeof(speedbuffer), "%dMHz", speed);
+ return speedbuffer;
+ }
+
+ if (speed%1000 == 0)
+ snprintf(speedbuffer, sizeof(speedbuffer),
+ "%dGHz", speed/1000);
+ else
+ snprintf(speedbuffer, sizeof(speedbuffer),
+ "%d.%dGHz", speed/1000, (speed%1000)/100);
+
+ return speedbuffer;
+}
+
+
+static unsigned int calc_speed(int mult)
+{
+ int khz;
+ khz = (mult/10)*fsb;
+ if (mult%10)
+ khz += fsb/2;
+ khz *= 1000;
+ return khz;
+}
+
+
+static int longhaul_get_cpu_mult(void)
+{
+ unsigned long invalue = 0, lo, hi;
+
+ rdmsr(MSR_IA32_EBL_CR_POWERON, lo, hi);
+ invalue = (lo & (1<<22|1<<23|1<<24|1<<25))>>22;
+ if (longhaul_version == TYPE_LONGHAUL_V2 ||
+ longhaul_version == TYPE_POWERSAVER) {
+ if (lo & (1<<27))
+ invalue += 16;
+ }
+ return eblcr[invalue];
+}
+
+/* For processor with BCR2 MSR */
+
+static void do_longhaul1(unsigned int mults_index)
+{
+ union msr_bcr2 bcr2;
+
+ rdmsrl(MSR_VIA_BCR2, bcr2.val);
+ /* Enable software clock multiplier */
+ bcr2.bits.ESOFTBF = 1;
+ bcr2.bits.CLOCKMUL = mults_index & 0xff;
+
+ /* Sync to timer tick */
+ safe_halt();
+ /* Change frequency on next halt or sleep */
+ wrmsrl(MSR_VIA_BCR2, bcr2.val);
+ /* Invoke transition */
+ ACPI_FLUSH_CPU_CACHE();
+ halt();
+
+ /* Disable software clock multiplier */
+ local_irq_disable();
+ rdmsrl(MSR_VIA_BCR2, bcr2.val);
+ bcr2.bits.ESOFTBF = 0;
+ wrmsrl(MSR_VIA_BCR2, bcr2.val);
+}
+
+/* For processor with Longhaul MSR */
+
+static void do_powersaver(int cx_address, unsigned int mults_index,
+ unsigned int dir)
+{
+ union msr_longhaul longhaul;
+ u32 t;
+
+ rdmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+ /* Setup new frequency */
+ if (!revid_errata)
+ longhaul.bits.RevisionKey = longhaul.bits.RevisionID;
+ else
+ longhaul.bits.RevisionKey = 0;
+ longhaul.bits.SoftBusRatio = mults_index & 0xf;
+ longhaul.bits.SoftBusRatio4 = (mults_index & 0x10) >> 4;
+ /* Setup new voltage */
+ if (can_scale_voltage)
+ longhaul.bits.SoftVID = (mults_index >> 8) & 0x1f;
+ /* Sync to timer tick */
+ safe_halt();
+ /* Raise voltage if necessary */
+ if (can_scale_voltage && dir) {
+ longhaul.bits.EnableSoftVID = 1;
+ wrmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+ /* Change voltage */
+ if (!cx_address) {
+ ACPI_FLUSH_CPU_CACHE();
+ halt();
+ } else {
+ ACPI_FLUSH_CPU_CACHE();
+ /* Invoke C3 */
+ inb(cx_address);
+ /* Dummy op - must do something useless after P_LVL3
+ * read */
+ t = inl(acpi_gbl_FADT.xpm_timer_block.address);
+ }
+ longhaul.bits.EnableSoftVID = 0;
+ wrmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+ }
+
+ /* Change frequency on next halt or sleep */
+ longhaul.bits.EnableSoftBusRatio = 1;
+ wrmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+ if (!cx_address) {
+ ACPI_FLUSH_CPU_CACHE();
+ halt();
+ } else {
+ ACPI_FLUSH_CPU_CACHE();
+ /* Invoke C3 */
+ inb(cx_address);
+ /* Dummy op - must do something useless after P_LVL3 read */
+ t = inl(acpi_gbl_FADT.xpm_timer_block.address);
+ }
+ /* Disable bus ratio bit */
+ longhaul.bits.EnableSoftBusRatio = 0;
+ wrmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+
+ /* Reduce voltage if necessary */
+ if (can_scale_voltage && !dir) {
+ longhaul.bits.EnableSoftVID = 1;
+ wrmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+ /* Change voltage */
+ if (!cx_address) {
+ ACPI_FLUSH_CPU_CACHE();
+ halt();
+ } else {
+ ACPI_FLUSH_CPU_CACHE();
+ /* Invoke C3 */
+ inb(cx_address);
+ /* Dummy op - must do something useless after P_LVL3
+ * read */
+ t = inl(acpi_gbl_FADT.xpm_timer_block.address);
+ }
+ longhaul.bits.EnableSoftVID = 0;
+ wrmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+ }
+}
+
+/**
+ * longhaul_set_cpu_frequency()
+ * @mults_index : bitpattern of the new multiplier.
+ *
+ * Sets a new clock ratio.
+ */
+
+static int longhaul_setstate(struct cpufreq_policy *policy,
+ unsigned int table_index)
+{
+ unsigned int mults_index;
+ int speed, mult;
+ struct cpufreq_freqs freqs;
+ unsigned long flags;
+ unsigned int pic1_mask, pic2_mask;
+ u16 bm_status = 0;
+ u32 bm_timeout = 1000;
+ unsigned int dir = 0;
+
+ mults_index = longhaul_table[table_index].driver_data;
+ /* Safety precautions */
+ mult = mults[mults_index & 0x1f];
+ if (mult == -1)
+ return -EINVAL;
+
+ speed = calc_speed(mult);
+ if ((speed > highest_speed) || (speed < lowest_speed))
+ return -EINVAL;
+
+ /* Voltage transition before frequency transition? */
+ if (can_scale_voltage && longhaul_index < table_index)
+ dir = 1;
+
+ freqs.old = calc_speed(longhaul_get_cpu_mult());
+ freqs.new = speed;
+
+ pr_debug("Setting to FSB:%dMHz Mult:%d.%dx (%s)\n",
+ fsb, mult/10, mult%10, print_speed(speed/1000));
+retry_loop:
+ preempt_disable();
+ local_irq_save(flags);
+
+ pic2_mask = inb(0xA1);
+ pic1_mask = inb(0x21); /* works on C3. save mask. */
+ outb(0xFF, 0xA1); /* Overkill */
+ outb(0xFE, 0x21); /* TMR0 only */
+
+ /* Wait while PCI bus is busy. */
+ if (acpi_regs_addr && (longhaul_flags & USE_NORTHBRIDGE
+ || ((pr != NULL) && pr->flags.bm_control))) {
+ bm_status = inw(acpi_regs_addr);
+ bm_status &= 1 << 4;
+ while (bm_status && bm_timeout) {
+ outw(1 << 4, acpi_regs_addr);
+ bm_timeout--;
+ bm_status = inw(acpi_regs_addr);
+ bm_status &= 1 << 4;
+ }
+ }
+
+ if (longhaul_flags & USE_NORTHBRIDGE) {
+ /* Disable AGP and PCI arbiters */
+ outb(3, 0x22);
+ } else if ((pr != NULL) && pr->flags.bm_control) {
+ /* Disable bus master arbitration */
+ acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 1);
+ }
+ switch (longhaul_version) {
+
+ /*
+ * Longhaul v1. (Samuel[C5A] and Samuel2 stepping 0[C5B])
+ * Software controlled multipliers only.
+ */
+ case TYPE_LONGHAUL_V1:
+ do_longhaul1(mults_index);
+ break;
+
+ /*
+ * Longhaul v2 appears in Samuel2 Steppings 1->7 [C5B] and Ezra [C5C]
+ *
+ * Longhaul v3 (aka Powersaver). (Ezra-T [C5M] & Nehemiah [C5N])
+ * Nehemiah can do FSB scaling too, but this has never been proven
+ * to work in practice.
+ */
+ case TYPE_LONGHAUL_V2:
+ case TYPE_POWERSAVER:
+ if (longhaul_flags & USE_ACPI_C3) {
+ /* Don't allow wakeup */
+ acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_RLD, 0);
+ do_powersaver(cx->address, mults_index, dir);
+ } else {
+ do_powersaver(0, mults_index, dir);
+ }
+ break;
+ }
+
+ if (longhaul_flags & USE_NORTHBRIDGE) {
+ /* Enable arbiters */
+ outb(0, 0x22);
+ } else if ((pr != NULL) && pr->flags.bm_control) {
+ /* Enable bus master arbitration */
+ acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 0);
+ }
+ outb(pic2_mask, 0xA1); /* restore mask */
+ outb(pic1_mask, 0x21);
+
+ local_irq_restore(flags);
+ preempt_enable();
+
+ freqs.new = calc_speed(longhaul_get_cpu_mult());
+ /* Check if requested frequency is set. */
+ if (unlikely(freqs.new != speed)) {
+ pr_info("Failed to set requested frequency!\n");
+ /* Revision ID = 1 but processor is expecting revision key
+ * equal to 0. Jumpers at the bottom of processor will change
+ * multiplier and FSB, but will not change bits in Longhaul
+ * MSR nor enable voltage scaling. */
+ if (!revid_errata) {
+ pr_info("Enabling \"Ignore Revision ID\" option\n");
+ revid_errata = 1;
+ msleep(200);
+ goto retry_loop;
+ }
+ /* Why ACPI C3 sometimes doesn't work is a mystery for me.
+ * But it does happen. Processor is entering ACPI C3 state,
+ * but it doesn't change frequency. I tried poking various
+ * bits in northbridge registers, but without success. */
+ if (longhaul_flags & USE_ACPI_C3) {
+ pr_info("Disabling ACPI C3 support\n");
+ longhaul_flags &= ~USE_ACPI_C3;
+ if (revid_errata) {
+ pr_info("Disabling \"Ignore Revision ID\" option\n");
+ revid_errata = 0;
+ }
+ msleep(200);
+ goto retry_loop;
+ }
+ /* This shouldn't happen. Longhaul ver. 2 was reported not
+ * working on processors without voltage scaling, but with
+ * RevID = 1. RevID errata will make things right. Just
+ * to be 100% sure. */
+ if (longhaul_version == TYPE_LONGHAUL_V2) {
+ pr_info("Switching to Longhaul ver. 1\n");
+ longhaul_version = TYPE_LONGHAUL_V1;
+ msleep(200);
+ goto retry_loop;
+ }
+ }
+
+ if (!bm_timeout) {
+ pr_info("Warning: Timeout while waiting for idle PCI bus\n");
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+/*
+ * Centaur decided to make life a little more tricky.
+ * Only longhaul v1 is allowed to read EBLCR BSEL[0:1].
+ * Samuel2 and above have to try and guess what the FSB is.
+ * We do this by assuming we booted at maximum multiplier, and interpolate
+ * between that value multiplied by possible FSBs and cpu_mhz which
+ * was calculated at boot time. Really ugly, but no other way to do this.
+ */
+
+#define ROUNDING 0xf
+
+static int guess_fsb(int mult)
+{
+ int speed = cpu_khz / 1000;
+ int i;
+ static const int speeds[] = { 666, 1000, 1333, 2000 };
+ int f_max, f_min;
+
+ for (i = 0; i < ARRAY_SIZE(speeds); i++) {
+ f_max = ((speeds[i] * mult) + 50) / 100;
+ f_max += (ROUNDING / 2);
+ f_min = f_max - ROUNDING;
+ if ((speed <= f_max) && (speed >= f_min))
+ return speeds[i] / 10;
+ }
+ return 0;
+}
+
+
+static int longhaul_get_ranges(void)
+{
+ unsigned int i, j, k = 0;
+ unsigned int ratio;
+ int mult;
+
+ /* Get current frequency */
+ mult = longhaul_get_cpu_mult();
+ if (mult == -1) {
+ pr_info("Invalid (reserved) multiplier!\n");
+ return -EINVAL;
+ }
+ fsb = guess_fsb(mult);
+ if (fsb == 0) {
+ pr_info("Invalid (reserved) FSB!\n");
+ return -EINVAL;
+ }
+ /* Get max multiplier - as we always did.
+ * Longhaul MSR is useful only when voltage scaling is enabled.
+ * C3 is booting at max anyway. */
+ maxmult = mult;
+ /* Get min multiplier */
+ switch (cpu_model) {
+ case CPU_NEHEMIAH:
+ minmult = 50;
+ break;
+ case CPU_NEHEMIAH_C:
+ minmult = 40;
+ break;
+ default:
+ minmult = 30;
+ break;
+ }
+
+ pr_debug("MinMult:%d.%dx MaxMult:%d.%dx\n",
+ minmult/10, minmult%10, maxmult/10, maxmult%10);
+
+ highest_speed = calc_speed(maxmult);
+ lowest_speed = calc_speed(minmult);
+ pr_debug("FSB:%dMHz Lowest speed: %s Highest speed:%s\n", fsb,
+ print_speed(lowest_speed/1000),
+ print_speed(highest_speed/1000));
+
+ if (lowest_speed == highest_speed) {
+ pr_info("highestspeed == lowest, aborting\n");
+ return -EINVAL;
+ }
+ if (lowest_speed > highest_speed) {
+ pr_info("nonsense! lowest (%d > %d) !\n",
+ lowest_speed, highest_speed);
+ return -EINVAL;
+ }
+
+ longhaul_table = kcalloc(numscales + 1, sizeof(*longhaul_table),
+ GFP_KERNEL);
+ if (!longhaul_table)
+ return -ENOMEM;
+
+ for (j = 0; j < numscales; j++) {
+ ratio = mults[j];
+ if (ratio == -1)
+ continue;
+ if (ratio > maxmult || ratio < minmult)
+ continue;
+ longhaul_table[k].frequency = calc_speed(ratio);
+ longhaul_table[k].driver_data = j;
+ k++;
+ }
+ if (k <= 1) {
+ kfree(longhaul_table);
+ return -ENODEV;
+ }
+ /* Sort */
+ for (j = 0; j < k - 1; j++) {
+ unsigned int min_f, min_i;
+ min_f = longhaul_table[j].frequency;
+ min_i = j;
+ for (i = j + 1; i < k; i++) {
+ if (longhaul_table[i].frequency < min_f) {
+ min_f = longhaul_table[i].frequency;
+ min_i = i;
+ }
+ }
+ if (min_i != j) {
+ swap(longhaul_table[j].frequency,
+ longhaul_table[min_i].frequency);
+ swap(longhaul_table[j].driver_data,
+ longhaul_table[min_i].driver_data);
+ }
+ }
+
+ longhaul_table[k].frequency = CPUFREQ_TABLE_END;
+
+ /* Find index we are running on */
+ for (j = 0; j < k; j++) {
+ if (mults[longhaul_table[j].driver_data & 0x1f] == mult) {
+ longhaul_index = j;
+ break;
+ }
+ }
+ return 0;
+}
+
+
+static void longhaul_setup_voltagescaling(void)
+{
+ struct cpufreq_frequency_table *freq_pos;
+ union msr_longhaul longhaul;
+ struct mV_pos minvid, maxvid, vid;
+ unsigned int j, speed, pos, kHz_step, numvscales;
+ int min_vid_speed;
+
+ rdmsrl(MSR_VIA_LONGHAUL, longhaul.val);
+ if (!(longhaul.bits.RevisionID & 1)) {
+ pr_info("Voltage scaling not supported by CPU\n");
+ return;
+ }
+
+ if (!longhaul.bits.VRMRev) {
+ pr_info("VRM 8.5\n");
+ vrm_mV_table = &vrm85_mV[0];
+ mV_vrm_table = &mV_vrm85[0];
+ } else {
+ pr_info("Mobile VRM\n");
+ if (cpu_model < CPU_NEHEMIAH)
+ return;
+ vrm_mV_table = &mobilevrm_mV[0];
+ mV_vrm_table = &mV_mobilevrm[0];
+ }
+
+ minvid = vrm_mV_table[longhaul.bits.MinimumVID];
+ maxvid = vrm_mV_table[longhaul.bits.MaximumVID];
+
+ if (minvid.mV == 0 || maxvid.mV == 0 || minvid.mV > maxvid.mV) {
+ pr_info("Bogus values Min:%d.%03d Max:%d.%03d - Voltage scaling disabled\n",
+ minvid.mV/1000, minvid.mV%1000,
+ maxvid.mV/1000, maxvid.mV%1000);
+ return;
+ }
+
+ if (minvid.mV == maxvid.mV) {
+ pr_info("Claims to support voltage scaling but min & max are both %d.%03d - Voltage scaling disabled\n",
+ maxvid.mV/1000, maxvid.mV%1000);
+ return;
+ }
+
+ /* How many voltage steps*/
+ numvscales = maxvid.pos - minvid.pos + 1;
+ pr_info("Max VID=%d.%03d Min VID=%d.%03d, %d possible voltage scales\n",
+ maxvid.mV/1000, maxvid.mV%1000,
+ minvid.mV/1000, minvid.mV%1000,
+ numvscales);
+
+ /* Calculate max frequency at min voltage */
+ j = longhaul.bits.MinMHzBR;
+ if (longhaul.bits.MinMHzBR4)
+ j += 16;
+ min_vid_speed = eblcr[j];
+ if (min_vid_speed == -1)
+ return;
+ switch (longhaul.bits.MinMHzFSB) {
+ case 0:
+ min_vid_speed *= 13333;
+ break;
+ case 1:
+ min_vid_speed *= 10000;
+ break;
+ case 3:
+ min_vid_speed *= 6666;
+ break;
+ default:
+ return;
+ }
+ if (min_vid_speed >= highest_speed)
+ return;
+ /* Calculate kHz for one voltage step */
+ kHz_step = (highest_speed - min_vid_speed) / numvscales;
+
+ cpufreq_for_each_entry_idx(freq_pos, longhaul_table, j) {
+ speed = freq_pos->frequency;
+ if (speed > min_vid_speed)
+ pos = (speed - min_vid_speed) / kHz_step + minvid.pos;
+ else
+ pos = minvid.pos;
+ freq_pos->driver_data |= mV_vrm_table[pos] << 8;
+ vid = vrm_mV_table[mV_vrm_table[pos]];
+ pr_info("f: %d kHz, index: %d, vid: %d mV\n",
+ speed, j, vid.mV);
+ }
+
+ can_scale_voltage = 1;
+ pr_info("Voltage scaling enabled\n");
+}
+
+
+static int longhaul_target(struct cpufreq_policy *policy,
+ unsigned int table_index)
+{
+ unsigned int i;
+ unsigned int dir = 0;
+ u8 vid, current_vid;
+ int retval = 0;
+
+ if (!can_scale_voltage)
+ retval = longhaul_setstate(policy, table_index);
+ else {
+ /* On test system voltage transitions exceeding single
+ * step up or down were turning motherboard off. Both
+ * "ondemand" and "userspace" are unsafe. C7 is doing
+ * this in hardware, C3 is old and we need to do this
+ * in software. */
+ i = longhaul_index;
+ current_vid = (longhaul_table[longhaul_index].driver_data >> 8);
+ current_vid &= 0x1f;
+ if (table_index > longhaul_index)
+ dir = 1;
+ while (i != table_index) {
+ vid = (longhaul_table[i].driver_data >> 8) & 0x1f;
+ if (vid != current_vid) {
+ retval = longhaul_setstate(policy, i);
+ current_vid = vid;
+ msleep(200);
+ }
+ if (dir)
+ i++;
+ else
+ i--;
+ }
+ retval = longhaul_setstate(policy, table_index);
+ }
+
+ longhaul_index = table_index;
+ return retval;
+}
+
+
+static unsigned int longhaul_get(unsigned int cpu)
+{
+ if (cpu)
+ return 0;
+ return calc_speed(longhaul_get_cpu_mult());
+}
+
+static acpi_status longhaul_walk_callback(acpi_handle obj_handle,
+ u32 nesting_level,
+ void *context, void **return_value)
+{
+ struct acpi_device *d = acpi_fetch_acpi_dev(obj_handle);
+
+ if (!d)
+ return 0;
+
+ *return_value = acpi_driver_data(d);
+ return 1;
+}
+
+/* VIA don't support PM2 reg, but have something similar */
+static int enable_arbiter_disable(void)
+{
+ struct pci_dev *dev;
+ int status = 1;
+ int reg;
+ u8 pci_cmd;
+
+ /* Find PLE133 host bridge */
+ reg = 0x78;
+ dev = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8601_0,
+ NULL);
+ /* Find PM133/VT8605 host bridge */
+ if (dev == NULL)
+ dev = pci_get_device(PCI_VENDOR_ID_VIA,
+ PCI_DEVICE_ID_VIA_8605_0, NULL);
+ /* Find CLE266 host bridge */
+ if (dev == NULL) {
+ reg = 0x76;
+ dev = pci_get_device(PCI_VENDOR_ID_VIA,
+ PCI_DEVICE_ID_VIA_862X_0, NULL);
+ /* Find CN400 V-Link host bridge */
+ if (dev == NULL)
+ dev = pci_get_device(PCI_VENDOR_ID_VIA, 0x7259, NULL);
+ }
+ if (dev != NULL) {
+ /* Enable access to port 0x22 */
+ pci_read_config_byte(dev, reg, &pci_cmd);
+ if (!(pci_cmd & 1<<7)) {
+ pci_cmd |= 1<<7;
+ pci_write_config_byte(dev, reg, pci_cmd);
+ pci_read_config_byte(dev, reg, &pci_cmd);
+ if (!(pci_cmd & 1<<7)) {
+ pr_err("Can't enable access to port 0x22\n");
+ status = 0;
+ }
+ }
+ pci_dev_put(dev);
+ return status;
+ }
+ return 0;
+}
+
+static int longhaul_setup_southbridge(void)
+{
+ struct pci_dev *dev;
+ u8 pci_cmd;
+
+ /* Find VT8235 southbridge */
+ dev = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8235, NULL);
+ if (dev == NULL)
+ /* Find VT8237 southbridge */
+ dev = pci_get_device(PCI_VENDOR_ID_VIA,
+ PCI_DEVICE_ID_VIA_8237, NULL);
+ if (dev != NULL) {
+ /* Set transition time to max */
+ pci_read_config_byte(dev, 0xec, &pci_cmd);
+ pci_cmd &= ~(1 << 2);
+ pci_write_config_byte(dev, 0xec, pci_cmd);
+ pci_read_config_byte(dev, 0xe4, &pci_cmd);
+ pci_cmd &= ~(1 << 7);
+ pci_write_config_byte(dev, 0xe4, pci_cmd);
+ pci_read_config_byte(dev, 0xe5, &pci_cmd);
+ pci_cmd |= 1 << 7;
+ pci_write_config_byte(dev, 0xe5, pci_cmd);
+ /* Get address of ACPI registers block*/
+ pci_read_config_byte(dev, 0x81, &pci_cmd);
+ if (pci_cmd & 1 << 7) {
+ pci_read_config_dword(dev, 0x88, &acpi_regs_addr);
+ acpi_regs_addr &= 0xff00;
+ pr_info("ACPI I/O at 0x%x\n", acpi_regs_addr);
+ }
+
+ pci_dev_put(dev);
+ return 1;
+ }
+ return 0;
+}
+
+static int longhaul_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpuinfo_x86 *c = &cpu_data(0);
+ char *cpuname = NULL;
+ int ret;
+ u32 lo, hi;
+
+ /* Check what we have on this motherboard */
+ switch (c->x86_model) {
+ case 6:
+ cpu_model = CPU_SAMUEL;
+ cpuname = "C3 'Samuel' [C5A]";
+ longhaul_version = TYPE_LONGHAUL_V1;
+ memcpy(mults, samuel1_mults, sizeof(samuel1_mults));
+ memcpy(eblcr, samuel1_eblcr, sizeof(samuel1_eblcr));
+ break;
+
+ case 7:
+ switch (c->x86_stepping) {
+ case 0:
+ longhaul_version = TYPE_LONGHAUL_V1;
+ cpu_model = CPU_SAMUEL2;
+ cpuname = "C3 'Samuel 2' [C5B]";
+ /* Note, this is not a typo, early Samuel2's had
+ * Samuel1 ratios. */
+ memcpy(mults, samuel1_mults, sizeof(samuel1_mults));
+ memcpy(eblcr, samuel2_eblcr, sizeof(samuel2_eblcr));
+ break;
+ case 1 ... 15:
+ longhaul_version = TYPE_LONGHAUL_V2;
+ if (c->x86_stepping < 8) {
+ cpu_model = CPU_SAMUEL2;
+ cpuname = "C3 'Samuel 2' [C5B]";
+ } else {
+ cpu_model = CPU_EZRA;
+ cpuname = "C3 'Ezra' [C5C]";
+ }
+ memcpy(mults, ezra_mults, sizeof(ezra_mults));
+ memcpy(eblcr, ezra_eblcr, sizeof(ezra_eblcr));
+ break;
+ }
+ break;
+
+ case 8:
+ cpu_model = CPU_EZRA_T;
+ cpuname = "C3 'Ezra-T' [C5M]";
+ longhaul_version = TYPE_POWERSAVER;
+ numscales = 32;
+ memcpy(mults, ezrat_mults, sizeof(ezrat_mults));
+ memcpy(eblcr, ezrat_eblcr, sizeof(ezrat_eblcr));
+ break;
+
+ case 9:
+ longhaul_version = TYPE_POWERSAVER;
+ numscales = 32;
+ memcpy(mults, nehemiah_mults, sizeof(nehemiah_mults));
+ memcpy(eblcr, nehemiah_eblcr, sizeof(nehemiah_eblcr));
+ switch (c->x86_stepping) {
+ case 0 ... 1:
+ cpu_model = CPU_NEHEMIAH;
+ cpuname = "C3 'Nehemiah A' [C5XLOE]";
+ break;
+ case 2 ... 4:
+ cpu_model = CPU_NEHEMIAH;
+ cpuname = "C3 'Nehemiah B' [C5XLOH]";
+ break;
+ case 5 ... 15:
+ cpu_model = CPU_NEHEMIAH_C;
+ cpuname = "C3 'Nehemiah C' [C5P]";
+ break;
+ }
+ break;
+
+ default:
+ cpuname = "Unknown";
+ break;
+ }
+ /* Check Longhaul ver. 2 */
+ if (longhaul_version == TYPE_LONGHAUL_V2) {
+ rdmsr(MSR_VIA_LONGHAUL, lo, hi);
+ if (lo == 0 && hi == 0)
+ /* Looks like MSR isn't present */
+ longhaul_version = TYPE_LONGHAUL_V1;
+ }
+
+ pr_info("VIA %s CPU detected. ", cpuname);
+ switch (longhaul_version) {
+ case TYPE_LONGHAUL_V1:
+ case TYPE_LONGHAUL_V2:
+ pr_cont("Longhaul v%d supported\n", longhaul_version);
+ break;
+ case TYPE_POWERSAVER:
+ pr_cont("Powersaver supported\n");
+ break;
+ }
+
+ /* Doesn't hurt */
+ longhaul_setup_southbridge();
+
+ /* Find ACPI data for processor */
+ acpi_walk_namespace(ACPI_TYPE_PROCESSOR, ACPI_ROOT_OBJECT,
+ ACPI_UINT32_MAX, &longhaul_walk_callback, NULL,
+ NULL, (void *)&pr);
+
+ /* Check ACPI support for C3 state */
+ if (pr != NULL && longhaul_version == TYPE_POWERSAVER) {
+ cx = &pr->power.states[ACPI_STATE_C3];
+ if (cx->address > 0 && cx->latency <= 1000)
+ longhaul_flags |= USE_ACPI_C3;
+ }
+ /* Disable if it isn't working */
+ if (disable_acpi_c3)
+ longhaul_flags &= ~USE_ACPI_C3;
+ /* Check if northbridge is friendly */
+ if (enable_arbiter_disable())
+ longhaul_flags |= USE_NORTHBRIDGE;
+
+ /* Check ACPI support for bus master arbiter disable */
+ if (!(longhaul_flags & USE_ACPI_C3
+ || longhaul_flags & USE_NORTHBRIDGE)
+ && ((pr == NULL) || !(pr->flags.bm_control))) {
+ pr_err("No ACPI support: Unsupported northbridge\n");
+ return -ENODEV;
+ }
+
+ if (longhaul_flags & USE_NORTHBRIDGE)
+ pr_info("Using northbridge support\n");
+ if (longhaul_flags & USE_ACPI_C3)
+ pr_info("Using ACPI support\n");
+
+ ret = longhaul_get_ranges();
+ if (ret != 0)
+ return ret;
+
+ if ((longhaul_version != TYPE_LONGHAUL_V1) && (scale_voltage != 0))
+ longhaul_setup_voltagescaling();
+
+ policy->transition_delay_us = 200000; /* usec */
+ policy->freq_table = longhaul_table;
+
+ return 0;
+}
+
+static struct cpufreq_driver longhaul_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = longhaul_target,
+ .get = longhaul_get,
+ .init = longhaul_cpu_init,
+ .name = "longhaul",
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct x86_cpu_id longhaul_id[] = {
+ X86_MATCH_VENDOR_FAM(CENTAUR, 6, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, longhaul_id);
+
+static int __init longhaul_init(void)
+{
+ struct cpuinfo_x86 *c = &cpu_data(0);
+
+ if (!x86_match_cpu(longhaul_id))
+ return -ENODEV;
+
+ if (!enable) {
+ pr_err("Option \"enable\" not set - Aborting\n");
+ return -ENODEV;
+ }
+#ifdef CONFIG_SMP
+ if (num_online_cpus() > 1) {
+ pr_err("More than 1 CPU detected, longhaul disabled\n");
+ return -ENODEV;
+ }
+#endif
+#ifdef CONFIG_X86_IO_APIC
+ if (boot_cpu_has(X86_FEATURE_APIC)) {
+ pr_err("APIC detected. Longhaul is currently broken in this configuration.\n");
+ return -ENODEV;
+ }
+#endif
+ switch (c->x86_model) {
+ case 6 ... 9:
+ return cpufreq_register_driver(&longhaul_driver);
+ case 10:
+ pr_err("Use acpi-cpufreq driver for VIA C7\n");
+ }
+
+ return -ENODEV;
+}
+
+
+static void __exit longhaul_exit(void)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get(0);
+ int i;
+
+ for (i = 0; i < numscales; i++) {
+ if (mults[i] == maxmult) {
+ struct cpufreq_freqs freqs;
+
+ freqs.old = policy->cur;
+ freqs.new = longhaul_table[i].frequency;
+ freqs.flags = 0;
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+ longhaul_setstate(policy, i);
+ cpufreq_freq_transition_end(policy, &freqs, 0);
+ break;
+ }
+ }
+
+ cpufreq_cpu_put(policy);
+ cpufreq_unregister_driver(&longhaul_driver);
+ kfree(longhaul_table);
+}
+
+/* Even if BIOS is exporting ACPI C3 state, and it is used
+ * with success when CPU is idle, this state doesn't
+ * trigger frequency transition in some cases. */
+module_param(disable_acpi_c3, int, 0644);
+MODULE_PARM_DESC(disable_acpi_c3, "Don't use ACPI C3 support");
+/* Change CPU voltage with frequency. Very useful to save
+ * power, but most VIA C3 processors aren't supporting it. */
+module_param(scale_voltage, int, 0644);
+MODULE_PARM_DESC(scale_voltage, "Scale voltage of processor");
+/* Force revision key to 0 for processors which doesn't
+ * support voltage scaling, but are introducing itself as
+ * such. */
+module_param(revid_errata, int, 0644);
+MODULE_PARM_DESC(revid_errata, "Ignore CPU Revision ID");
+/* By default driver is disabled to prevent incompatible
+ * system freeze. */
+module_param(enable, int, 0644);
+MODULE_PARM_DESC(enable, "Enable driver");
+
+MODULE_AUTHOR("Dave Jones");
+MODULE_DESCRIPTION("Longhaul driver for VIA Cyrix processors.");
+MODULE_LICENSE("GPL");
+
+late_initcall(longhaul_init);
+module_exit(longhaul_exit);
diff --git a/drivers/cpufreq/longhaul.h b/drivers/cpufreq/longhaul.h
new file mode 100644
index 0000000000..89c4cc297c
--- /dev/null
+++ b/drivers/cpufreq/longhaul.h
@@ -0,0 +1,352 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * longhaul.h
+ * (C) 2003 Dave Jones.
+ *
+ * VIA-specific information
+ */
+
+union msr_bcr2 {
+ struct {
+ unsigned Reseved:19, // 18:0
+ ESOFTBF:1, // 19
+ Reserved2:3, // 22:20
+ CLOCKMUL:4, // 26:23
+ Reserved3:5; // 31:27
+ } bits;
+ unsigned long val;
+};
+
+union msr_longhaul {
+ struct {
+ unsigned RevisionID:4, // 3:0
+ RevisionKey:4, // 7:4
+ EnableSoftBusRatio:1, // 8
+ EnableSoftVID:1, // 9
+ EnableSoftBSEL:1, // 10
+ Reserved:3, // 11:13
+ SoftBusRatio4:1, // 14
+ VRMRev:1, // 15
+ SoftBusRatio:4, // 19:16
+ SoftVID:5, // 24:20
+ Reserved2:3, // 27:25
+ SoftBSEL:2, // 29:28
+ Reserved3:2, // 31:30
+ MaxMHzBR:4, // 35:32
+ MaximumVID:5, // 40:36
+ MaxMHzFSB:2, // 42:41
+ MaxMHzBR4:1, // 43
+ Reserved4:4, // 47:44
+ MinMHzBR:4, // 51:48
+ MinimumVID:5, // 56:52
+ MinMHzFSB:2, // 58:57
+ MinMHzBR4:1, // 59
+ Reserved5:4; // 63:60
+ } bits;
+ unsigned long long val;
+};
+
+/*
+ * Clock ratio tables. Div/Mod by 10 to get ratio.
+ * The eblcr values specify the ratio read from the CPU.
+ * The mults values specify what to write to the CPU.
+ */
+
+/*
+ * VIA C3 Samuel 1 & Samuel 2 (stepping 0)
+ */
+static const int samuel1_mults[16] = {
+ -1, /* 0000 -> RESERVED */
+ 30, /* 0001 -> 3.0x */
+ 40, /* 0010 -> 4.0x */
+ -1, /* 0011 -> RESERVED */
+ -1, /* 0100 -> RESERVED */
+ 35, /* 0101 -> 3.5x */
+ 45, /* 0110 -> 4.5x */
+ 55, /* 0111 -> 5.5x */
+ 60, /* 1000 -> 6.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 50, /* 1011 -> 5.0x */
+ 65, /* 1100 -> 6.5x */
+ 75, /* 1101 -> 7.5x */
+ -1, /* 1110 -> RESERVED */
+ -1, /* 1111 -> RESERVED */
+};
+
+static const int samuel1_eblcr[16] = {
+ 50, /* 0000 -> RESERVED */
+ 30, /* 0001 -> 3.0x */
+ 40, /* 0010 -> 4.0x */
+ -1, /* 0011 -> RESERVED */
+ 55, /* 0100 -> 5.5x */
+ 35, /* 0101 -> 3.5x */
+ 45, /* 0110 -> 4.5x */
+ -1, /* 0111 -> RESERVED */
+ -1, /* 1000 -> RESERVED */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 60, /* 1011 -> 6.0x */
+ -1, /* 1100 -> RESERVED */
+ 75, /* 1101 -> 7.5x */
+ -1, /* 1110 -> RESERVED */
+ 65, /* 1111 -> 6.5x */
+};
+
+/*
+ * VIA C3 Samuel2 Stepping 1->15
+ */
+static const int samuel2_eblcr[16] = {
+ 50, /* 0000 -> 5.0x */
+ 30, /* 0001 -> 3.0x */
+ 40, /* 0010 -> 4.0x */
+ 100, /* 0011 -> 10.0x */
+ 55, /* 0100 -> 5.5x */
+ 35, /* 0101 -> 3.5x */
+ 45, /* 0110 -> 4.5x */
+ 110, /* 0111 -> 11.0x */
+ 90, /* 1000 -> 9.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 60, /* 1011 -> 6.0x */
+ 120, /* 1100 -> 12.0x */
+ 75, /* 1101 -> 7.5x */
+ 130, /* 1110 -> 13.0x */
+ 65, /* 1111 -> 6.5x */
+};
+
+/*
+ * VIA C3 Ezra
+ */
+static const int ezra_mults[16] = {
+ 100, /* 0000 -> 10.0x */
+ 30, /* 0001 -> 3.0x */
+ 40, /* 0010 -> 4.0x */
+ 90, /* 0011 -> 9.0x */
+ 95, /* 0100 -> 9.5x */
+ 35, /* 0101 -> 3.5x */
+ 45, /* 0110 -> 4.5x */
+ 55, /* 0111 -> 5.5x */
+ 60, /* 1000 -> 6.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 50, /* 1011 -> 5.0x */
+ 65, /* 1100 -> 6.5x */
+ 75, /* 1101 -> 7.5x */
+ 85, /* 1110 -> 8.5x */
+ 120, /* 1111 -> 12.0x */
+};
+
+static const int ezra_eblcr[16] = {
+ 50, /* 0000 -> 5.0x */
+ 30, /* 0001 -> 3.0x */
+ 40, /* 0010 -> 4.0x */
+ 100, /* 0011 -> 10.0x */
+ 55, /* 0100 -> 5.5x */
+ 35, /* 0101 -> 3.5x */
+ 45, /* 0110 -> 4.5x */
+ 95, /* 0111 -> 9.5x */
+ 90, /* 1000 -> 9.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 60, /* 1011 -> 6.0x */
+ 120, /* 1100 -> 12.0x */
+ 75, /* 1101 -> 7.5x */
+ 85, /* 1110 -> 8.5x */
+ 65, /* 1111 -> 6.5x */
+};
+
+/*
+ * VIA C3 (Ezra-T) [C5M].
+ */
+static const int ezrat_mults[32] = {
+ 100, /* 0000 -> 10.0x */
+ 30, /* 0001 -> 3.0x */
+ 40, /* 0010 -> 4.0x */
+ 90, /* 0011 -> 9.0x */
+ 95, /* 0100 -> 9.5x */
+ 35, /* 0101 -> 3.5x */
+ 45, /* 0110 -> 4.5x */
+ 55, /* 0111 -> 5.5x */
+ 60, /* 1000 -> 6.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 50, /* 1011 -> 5.0x */
+ 65, /* 1100 -> 6.5x */
+ 75, /* 1101 -> 7.5x */
+ 85, /* 1110 -> 8.5x */
+ 120, /* 1111 -> 12.0x */
+
+ -1, /* 0000 -> RESERVED (10.0x) */
+ 110, /* 0001 -> 11.0x */
+ -1, /* 0010 -> 12.0x */
+ -1, /* 0011 -> RESERVED (9.0x)*/
+ 105, /* 0100 -> 10.5x */
+ 115, /* 0101 -> 11.5x */
+ 125, /* 0110 -> 12.5x */
+ 135, /* 0111 -> 13.5x */
+ 140, /* 1000 -> 14.0x */
+ 150, /* 1001 -> 15.0x */
+ 160, /* 1010 -> 16.0x */
+ 130, /* 1011 -> 13.0x */
+ 145, /* 1100 -> 14.5x */
+ 155, /* 1101 -> 15.5x */
+ -1, /* 1110 -> RESERVED (13.0x) */
+ -1, /* 1111 -> RESERVED (12.0x) */
+};
+
+static const int ezrat_eblcr[32] = {
+ 50, /* 0000 -> 5.0x */
+ 30, /* 0001 -> 3.0x */
+ 40, /* 0010 -> 4.0x */
+ 100, /* 0011 -> 10.0x */
+ 55, /* 0100 -> 5.5x */
+ 35, /* 0101 -> 3.5x */
+ 45, /* 0110 -> 4.5x */
+ 95, /* 0111 -> 9.5x */
+ 90, /* 1000 -> 9.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 60, /* 1011 -> 6.0x */
+ 120, /* 1100 -> 12.0x */
+ 75, /* 1101 -> 7.5x */
+ 85, /* 1110 -> 8.5x */
+ 65, /* 1111 -> 6.5x */
+
+ -1, /* 0000 -> RESERVED (9.0x) */
+ 110, /* 0001 -> 11.0x */
+ 120, /* 0010 -> 12.0x */
+ -1, /* 0011 -> RESERVED (10.0x)*/
+ 135, /* 0100 -> 13.5x */
+ 115, /* 0101 -> 11.5x */
+ 125, /* 0110 -> 12.5x */
+ 105, /* 0111 -> 10.5x */
+ 130, /* 1000 -> 13.0x */
+ 150, /* 1001 -> 15.0x */
+ 160, /* 1010 -> 16.0x */
+ 140, /* 1011 -> 14.0x */
+ -1, /* 1100 -> RESERVED (12.0x) */
+ 155, /* 1101 -> 15.5x */
+ -1, /* 1110 -> RESERVED (13.0x) */
+ 145, /* 1111 -> 14.5x */
+};
+
+/*
+ * VIA C3 Nehemiah */
+
+static const int nehemiah_mults[32] = {
+ 100, /* 0000 -> 10.0x */
+ -1, /* 0001 -> 16.0x */
+ 40, /* 0010 -> 4.0x */
+ 90, /* 0011 -> 9.0x */
+ 95, /* 0100 -> 9.5x */
+ -1, /* 0101 -> RESERVED */
+ 45, /* 0110 -> 4.5x */
+ 55, /* 0111 -> 5.5x */
+ 60, /* 1000 -> 6.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 50, /* 1011 -> 5.0x */
+ 65, /* 1100 -> 6.5x */
+ 75, /* 1101 -> 7.5x */
+ 85, /* 1110 -> 8.5x */
+ 120, /* 1111 -> 12.0x */
+ -1, /* 0000 -> 10.0x */
+ 110, /* 0001 -> 11.0x */
+ -1, /* 0010 -> 12.0x */
+ -1, /* 0011 -> 9.0x */
+ 105, /* 0100 -> 10.5x */
+ 115, /* 0101 -> 11.5x */
+ 125, /* 0110 -> 12.5x */
+ 135, /* 0111 -> 13.5x */
+ 140, /* 1000 -> 14.0x */
+ 150, /* 1001 -> 15.0x */
+ 160, /* 1010 -> 16.0x */
+ 130, /* 1011 -> 13.0x */
+ 145, /* 1100 -> 14.5x */
+ 155, /* 1101 -> 15.5x */
+ -1, /* 1110 -> RESERVED (13.0x) */
+ -1, /* 1111 -> 12.0x */
+};
+
+static const int nehemiah_eblcr[32] = {
+ 50, /* 0000 -> 5.0x */
+ 160, /* 0001 -> 16.0x */
+ 40, /* 0010 -> 4.0x */
+ 100, /* 0011 -> 10.0x */
+ 55, /* 0100 -> 5.5x */
+ -1, /* 0101 -> RESERVED */
+ 45, /* 0110 -> 4.5x */
+ 95, /* 0111 -> 9.5x */
+ 90, /* 1000 -> 9.0x */
+ 70, /* 1001 -> 7.0x */
+ 80, /* 1010 -> 8.0x */
+ 60, /* 1011 -> 6.0x */
+ 120, /* 1100 -> 12.0x */
+ 75, /* 1101 -> 7.5x */
+ 85, /* 1110 -> 8.5x */
+ 65, /* 1111 -> 6.5x */
+ 90, /* 0000 -> 9.0x */
+ 110, /* 0001 -> 11.0x */
+ 120, /* 0010 -> 12.0x */
+ 100, /* 0011 -> 10.0x */
+ 135, /* 0100 -> 13.5x */
+ 115, /* 0101 -> 11.5x */
+ 125, /* 0110 -> 12.5x */
+ 105, /* 0111 -> 10.5x */
+ 130, /* 1000 -> 13.0x */
+ 150, /* 1001 -> 15.0x */
+ 160, /* 1010 -> 16.0x */
+ 140, /* 1011 -> 14.0x */
+ 120, /* 1100 -> 12.0x */
+ 155, /* 1101 -> 15.5x */
+ -1, /* 1110 -> RESERVED (13.0x) */
+ 145 /* 1111 -> 14.5x */
+};
+
+/*
+ * Voltage scales. Div/Mod by 1000 to get actual voltage.
+ * Which scale to use depends on the VRM type in use.
+ */
+
+struct mV_pos {
+ unsigned short mV;
+ unsigned short pos;
+};
+
+static const struct mV_pos vrm85_mV[32] = {
+ {1250, 8}, {1200, 6}, {1150, 4}, {1100, 2},
+ {1050, 0}, {1800, 30}, {1750, 28}, {1700, 26},
+ {1650, 24}, {1600, 22}, {1550, 20}, {1500, 18},
+ {1450, 16}, {1400, 14}, {1350, 12}, {1300, 10},
+ {1275, 9}, {1225, 7}, {1175, 5}, {1125, 3},
+ {1075, 1}, {1825, 31}, {1775, 29}, {1725, 27},
+ {1675, 25}, {1625, 23}, {1575, 21}, {1525, 19},
+ {1475, 17}, {1425, 15}, {1375, 13}, {1325, 11}
+};
+
+static const unsigned char mV_vrm85[32] = {
+ 0x04, 0x14, 0x03, 0x13, 0x02, 0x12, 0x01, 0x11,
+ 0x00, 0x10, 0x0f, 0x1f, 0x0e, 0x1e, 0x0d, 0x1d,
+ 0x0c, 0x1c, 0x0b, 0x1b, 0x0a, 0x1a, 0x09, 0x19,
+ 0x08, 0x18, 0x07, 0x17, 0x06, 0x16, 0x05, 0x15
+};
+
+static const struct mV_pos mobilevrm_mV[32] = {
+ {1750, 31}, {1700, 30}, {1650, 29}, {1600, 28},
+ {1550, 27}, {1500, 26}, {1450, 25}, {1400, 24},
+ {1350, 23}, {1300, 22}, {1250, 21}, {1200, 20},
+ {1150, 19}, {1100, 18}, {1050, 17}, {1000, 16},
+ {975, 15}, {950, 14}, {925, 13}, {900, 12},
+ {875, 11}, {850, 10}, {825, 9}, {800, 8},
+ {775, 7}, {750, 6}, {725, 5}, {700, 4},
+ {675, 3}, {650, 2}, {625, 1}, {600, 0}
+};
+
+static const unsigned char mV_mobilevrm[32] = {
+ 0x1f, 0x1e, 0x1d, 0x1c, 0x1b, 0x1a, 0x19, 0x18,
+ 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10,
+ 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
+ 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
+};
+
diff --git a/drivers/cpufreq/longrun.c b/drivers/cpufreq/longrun.c
new file mode 100644
index 0000000000..1caaec7c28
--- /dev/null
+++ b/drivers/cpufreq/longrun.c
@@ -0,0 +1,317 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/timex.h>
+
+#include <asm/msr.h>
+#include <asm/processor.h>
+#include <asm/cpu_device_id.h>
+
+static struct cpufreq_driver longrun_driver;
+
+/**
+ * longrun_{low,high}_freq is needed for the conversion of cpufreq kHz
+ * values into per cent values. In TMTA microcode, the following is valid:
+ * performance_pctg = (current_freq - low_freq)/(high_freq - low_freq)
+ */
+static unsigned int longrun_low_freq, longrun_high_freq;
+
+
+/**
+ * longrun_get_policy - get the current LongRun policy
+ * @policy: struct cpufreq_policy where current policy is written into
+ *
+ * Reads the current LongRun policy by access to MSR_TMTA_LONGRUN_FLAGS
+ * and MSR_TMTA_LONGRUN_CTRL
+ */
+static void longrun_get_policy(struct cpufreq_policy *policy)
+{
+ u32 msr_lo, msr_hi;
+
+ rdmsr(MSR_TMTA_LONGRUN_FLAGS, msr_lo, msr_hi);
+ pr_debug("longrun flags are %x - %x\n", msr_lo, msr_hi);
+ if (msr_lo & 0x01)
+ policy->policy = CPUFREQ_POLICY_PERFORMANCE;
+ else
+ policy->policy = CPUFREQ_POLICY_POWERSAVE;
+
+ rdmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
+ pr_debug("longrun ctrl is %x - %x\n", msr_lo, msr_hi);
+ msr_lo &= 0x0000007F;
+ msr_hi &= 0x0000007F;
+
+ if (longrun_high_freq <= longrun_low_freq) {
+ /* Assume degenerate Longrun table */
+ policy->min = policy->max = longrun_high_freq;
+ } else {
+ policy->min = longrun_low_freq + msr_lo *
+ ((longrun_high_freq - longrun_low_freq) / 100);
+ policy->max = longrun_low_freq + msr_hi *
+ ((longrun_high_freq - longrun_low_freq) / 100);
+ }
+ policy->cpu = 0;
+}
+
+
+/**
+ * longrun_set_policy - sets a new CPUFreq policy
+ * @policy: new policy
+ *
+ * Sets a new CPUFreq policy on LongRun-capable processors. This function
+ * has to be called with cpufreq_driver locked.
+ */
+static int longrun_set_policy(struct cpufreq_policy *policy)
+{
+ u32 msr_lo, msr_hi;
+ u32 pctg_lo, pctg_hi;
+
+ if (!policy)
+ return -EINVAL;
+
+ if (longrun_high_freq <= longrun_low_freq) {
+ /* Assume degenerate Longrun table */
+ pctg_lo = pctg_hi = 100;
+ } else {
+ pctg_lo = (policy->min - longrun_low_freq) /
+ ((longrun_high_freq - longrun_low_freq) / 100);
+ pctg_hi = (policy->max - longrun_low_freq) /
+ ((longrun_high_freq - longrun_low_freq) / 100);
+ }
+
+ if (pctg_hi > 100)
+ pctg_hi = 100;
+ if (pctg_lo > pctg_hi)
+ pctg_lo = pctg_hi;
+
+ /* performance or economy mode */
+ rdmsr(MSR_TMTA_LONGRUN_FLAGS, msr_lo, msr_hi);
+ msr_lo &= 0xFFFFFFFE;
+ switch (policy->policy) {
+ case CPUFREQ_POLICY_PERFORMANCE:
+ msr_lo |= 0x00000001;
+ break;
+ case CPUFREQ_POLICY_POWERSAVE:
+ break;
+ }
+ wrmsr(MSR_TMTA_LONGRUN_FLAGS, msr_lo, msr_hi);
+
+ /* lower and upper boundary */
+ rdmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
+ msr_lo &= 0xFFFFFF80;
+ msr_hi &= 0xFFFFFF80;
+ msr_lo |= pctg_lo;
+ msr_hi |= pctg_hi;
+ wrmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
+
+ return 0;
+}
+
+
+/**
+ * longrun_verify_poliy - verifies a new CPUFreq policy
+ * @policy: the policy to verify
+ *
+ * Validates a new CPUFreq policy. This function has to be called with
+ * cpufreq_driver locked.
+ */
+static int longrun_verify_policy(struct cpufreq_policy_data *policy)
+{
+ if (!policy)
+ return -EINVAL;
+
+ policy->cpu = 0;
+ cpufreq_verify_within_cpu_limits(policy);
+
+ return 0;
+}
+
+static unsigned int longrun_get(unsigned int cpu)
+{
+ u32 eax, ebx, ecx, edx;
+
+ if (cpu)
+ return 0;
+
+ cpuid(0x80860007, &eax, &ebx, &ecx, &edx);
+ pr_debug("cpuid eax is %u\n", eax);
+
+ return eax * 1000;
+}
+
+/**
+ * longrun_determine_freqs - determines the lowest and highest possible core frequency
+ * @low_freq: an int to put the lowest frequency into
+ * @high_freq: an int to put the highest frequency into
+ *
+ * Determines the lowest and highest possible core frequencies on this CPU.
+ * This is necessary to calculate the performance percentage according to
+ * TMTA rules:
+ * performance_pctg = (target_freq - low_freq)/(high_freq - low_freq)
+ */
+static int longrun_determine_freqs(unsigned int *low_freq,
+ unsigned int *high_freq)
+{
+ u32 msr_lo, msr_hi;
+ u32 save_lo, save_hi;
+ u32 eax, ebx, ecx, edx;
+ u32 try_hi;
+ struct cpuinfo_x86 *c = &cpu_data(0);
+
+ if (!low_freq || !high_freq)
+ return -EINVAL;
+
+ if (cpu_has(c, X86_FEATURE_LRTI)) {
+ /* if the LongRun Table Interface is present, the
+ * detection is a bit easier:
+ * For minimum frequency, read out the maximum
+ * level (msr_hi), write that into "currently
+ * selected level", and read out the frequency.
+ * For maximum frequency, read out level zero.
+ */
+ /* minimum */
+ rdmsr(MSR_TMTA_LRTI_READOUT, msr_lo, msr_hi);
+ wrmsr(MSR_TMTA_LRTI_READOUT, msr_hi, msr_hi);
+ rdmsr(MSR_TMTA_LRTI_VOLT_MHZ, msr_lo, msr_hi);
+ *low_freq = msr_lo * 1000; /* to kHz */
+
+ /* maximum */
+ wrmsr(MSR_TMTA_LRTI_READOUT, 0, msr_hi);
+ rdmsr(MSR_TMTA_LRTI_VOLT_MHZ, msr_lo, msr_hi);
+ *high_freq = msr_lo * 1000; /* to kHz */
+
+ pr_debug("longrun table interface told %u - %u kHz\n",
+ *low_freq, *high_freq);
+
+ if (*low_freq > *high_freq)
+ *low_freq = *high_freq;
+ return 0;
+ }
+
+ /* set the upper border to the value determined during TSC init */
+ *high_freq = (cpu_khz / 1000);
+ *high_freq = *high_freq * 1000;
+ pr_debug("high frequency is %u kHz\n", *high_freq);
+
+ /* get current borders */
+ rdmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
+ save_lo = msr_lo & 0x0000007F;
+ save_hi = msr_hi & 0x0000007F;
+
+ /* if current perf_pctg is larger than 90%, we need to decrease the
+ * upper limit to make the calculation more accurate.
+ */
+ cpuid(0x80860007, &eax, &ebx, &ecx, &edx);
+ /* try decreasing in 10% steps, some processors react only
+ * on some barrier values */
+ for (try_hi = 80; try_hi > 0 && ecx > 90; try_hi -= 10) {
+ /* set to 0 to try_hi perf_pctg */
+ msr_lo &= 0xFFFFFF80;
+ msr_hi &= 0xFFFFFF80;
+ msr_hi |= try_hi;
+ wrmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
+
+ /* read out current core MHz and current perf_pctg */
+ cpuid(0x80860007, &eax, &ebx, &ecx, &edx);
+
+ /* restore values */
+ wrmsr(MSR_TMTA_LONGRUN_CTRL, save_lo, save_hi);
+ }
+ pr_debug("percentage is %u %%, freq is %u MHz\n", ecx, eax);
+
+ /* performance_pctg = (current_freq - low_freq)/(high_freq - low_freq)
+ * eqals
+ * low_freq * (1 - perf_pctg) = (cur_freq - high_freq * perf_pctg)
+ *
+ * high_freq * perf_pctg is stored tempoarily into "ebx".
+ */
+ ebx = (((cpu_khz / 1000) * ecx) / 100); /* to MHz */
+
+ if ((ecx > 95) || (ecx == 0) || (eax < ebx))
+ return -EIO;
+
+ edx = ((eax - ebx) * 100) / (100 - ecx);
+ *low_freq = edx * 1000; /* back to kHz */
+
+ pr_debug("low frequency is %u kHz\n", *low_freq);
+
+ if (*low_freq > *high_freq)
+ *low_freq = *high_freq;
+
+ return 0;
+}
+
+
+static int longrun_cpu_init(struct cpufreq_policy *policy)
+{
+ int result = 0;
+
+ /* capability check */
+ if (policy->cpu != 0)
+ return -ENODEV;
+
+ /* detect low and high frequency */
+ result = longrun_determine_freqs(&longrun_low_freq, &longrun_high_freq);
+ if (result)
+ return result;
+
+ /* cpuinfo and default policy values */
+ policy->cpuinfo.min_freq = longrun_low_freq;
+ policy->cpuinfo.max_freq = longrun_high_freq;
+ longrun_get_policy(policy);
+
+ return 0;
+}
+
+
+static struct cpufreq_driver longrun_driver = {
+ .flags = CPUFREQ_CONST_LOOPS,
+ .verify = longrun_verify_policy,
+ .setpolicy = longrun_set_policy,
+ .get = longrun_get,
+ .init = longrun_cpu_init,
+ .name = "longrun",
+};
+
+static const struct x86_cpu_id longrun_ids[] = {
+ X86_MATCH_VENDOR_FEATURE(TRANSMETA, X86_FEATURE_LONGRUN, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, longrun_ids);
+
+/**
+ * longrun_init - initializes the Transmeta Crusoe LongRun CPUFreq driver
+ *
+ * Initializes the LongRun support.
+ */
+static int __init longrun_init(void)
+{
+ if (!x86_match_cpu(longrun_ids))
+ return -ENODEV;
+ return cpufreq_register_driver(&longrun_driver);
+}
+
+
+/**
+ * longrun_exit - unregisters LongRun support
+ */
+static void __exit longrun_exit(void)
+{
+ cpufreq_unregister_driver(&longrun_driver);
+}
+
+
+MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>");
+MODULE_DESCRIPTION("LongRun driver for Transmeta Crusoe and "
+ "Efficeon processors.");
+MODULE_LICENSE("GPL");
+
+module_init(longrun_init);
+module_exit(longrun_exit);
diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c
new file mode 100644
index 0000000000..afc59b2921
--- /dev/null
+++ b/drivers/cpufreq/loongson2_cpufreq.c
@@ -0,0 +1,184 @@
+/*
+ * Cpufreq driver for the loongson-2 processors
+ *
+ * The 2E revision of loongson processor not support this feature.
+ *
+ * Copyright (C) 2006 - 2008 Lemote Inc. & Institute of Computing Technology
+ * Author: Yanhua, yanh@lemote.com
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/platform_device.h>
+
+#include <asm/idle.h>
+
+#include <asm/mach-loongson2ef/loongson.h>
+
+static uint nowait;
+
+static void (*saved_cpu_wait) (void);
+
+static int loongson2_cpu_freq_notifier(struct notifier_block *nb,
+ unsigned long val, void *data);
+
+static struct notifier_block loongson2_cpufreq_notifier_block = {
+ .notifier_call = loongson2_cpu_freq_notifier
+};
+
+static int loongson2_cpu_freq_notifier(struct notifier_block *nb,
+ unsigned long val, void *data)
+{
+ if (val == CPUFREQ_POSTCHANGE)
+ current_cpu_data.udelay_val = loops_per_jiffy;
+
+ return 0;
+}
+
+/*
+ * Here we notify other drivers of the proposed change and the final change.
+ */
+static int loongson2_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ unsigned int freq;
+
+ freq =
+ ((cpu_clock_freq / 1000) *
+ loongson2_clockmod_table[index].driver_data) / 8;
+
+ /* setting the cpu frequency */
+ loongson2_cpu_set_rate(freq);
+
+ return 0;
+}
+
+static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ int i;
+ unsigned long rate;
+ int ret;
+
+ rate = cpu_clock_freq / 1000;
+ if (!rate)
+ return -EINVAL;
+
+ /* clock table init */
+ for (i = 2;
+ (loongson2_clockmod_table[i].frequency != CPUFREQ_TABLE_END);
+ i++)
+ loongson2_clockmod_table[i].frequency = (rate * i) / 8;
+
+ ret = loongson2_cpu_set_rate(rate);
+ if (ret)
+ return ret;
+
+ cpufreq_generic_init(policy, &loongson2_clockmod_table[0], 0);
+ return 0;
+}
+
+static int loongson2_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ return 0;
+}
+
+static struct cpufreq_driver loongson2_cpufreq_driver = {
+ .name = "loongson2",
+ .init = loongson2_cpufreq_cpu_init,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = loongson2_cpufreq_target,
+ .get = cpufreq_generic_get,
+ .exit = loongson2_cpufreq_exit,
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct platform_device_id platform_device_ids[] = {
+ {
+ .name = "loongson2_cpufreq",
+ },
+ {}
+};
+
+MODULE_DEVICE_TABLE(platform, platform_device_ids);
+
+static struct platform_driver platform_driver = {
+ .driver = {
+ .name = "loongson2_cpufreq",
+ },
+ .id_table = platform_device_ids,
+};
+
+/*
+ * This is the simple version of Loongson-2 wait, Maybe we need do this in
+ * interrupt disabled context.
+ */
+
+static DEFINE_SPINLOCK(loongson2_wait_lock);
+
+static void loongson2_cpu_wait(void)
+{
+ unsigned long flags;
+ u32 cpu_freq;
+
+ spin_lock_irqsave(&loongson2_wait_lock, flags);
+ cpu_freq = readl(LOONGSON_CHIPCFG);
+ /* Put CPU into wait mode */
+ writel(readl(LOONGSON_CHIPCFG) & ~0x7, LOONGSON_CHIPCFG);
+ /* Restore CPU state */
+ writel(cpu_freq, LOONGSON_CHIPCFG);
+ spin_unlock_irqrestore(&loongson2_wait_lock, flags);
+ local_irq_enable();
+}
+
+static int __init cpufreq_init(void)
+{
+ int ret;
+
+ /* Register platform stuff */
+ ret = platform_driver_register(&platform_driver);
+ if (ret)
+ return ret;
+
+ pr_info("Loongson-2F CPU frequency driver\n");
+
+ cpufreq_register_notifier(&loongson2_cpufreq_notifier_block,
+ CPUFREQ_TRANSITION_NOTIFIER);
+
+ ret = cpufreq_register_driver(&loongson2_cpufreq_driver);
+
+ if (!ret && !nowait) {
+ saved_cpu_wait = cpu_wait;
+ cpu_wait = loongson2_cpu_wait;
+ }
+
+ return ret;
+}
+
+static void __exit cpufreq_exit(void)
+{
+ if (!nowait && saved_cpu_wait)
+ cpu_wait = saved_cpu_wait;
+ cpufreq_unregister_driver(&loongson2_cpufreq_driver);
+ cpufreq_unregister_notifier(&loongson2_cpufreq_notifier_block,
+ CPUFREQ_TRANSITION_NOTIFIER);
+
+ platform_driver_unregister(&platform_driver);
+}
+
+module_init(cpufreq_init);
+module_exit(cpufreq_exit);
+
+module_param(nowait, uint, 0644);
+MODULE_PARM_DESC(nowait, "Disable Loongson-2F specific wait");
+
+MODULE_AUTHOR("Yanhua <yanh@lemote.com>");
+MODULE_DESCRIPTION("cpufreq driver for Loongson2F");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/maple-cpufreq.c b/drivers/cpufreq/maple-cpufreq.c
new file mode 100644
index 0000000000..f9306410a0
--- /dev/null
+++ b/drivers/cpufreq/maple-cpufreq.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2011 Dmitry Eremin-Solenikov
+ * Copyright (C) 2002 - 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>
+ * and Markus Demleitner <msdemlei@cl.uni-heidelberg.de>
+ *
+ * This driver adds basic cpufreq support for SMU & 970FX based G5 Macs,
+ * that is iMac G5 and latest single CPU desktop.
+ */
+
+#undef DEBUG
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/completion.h>
+#include <linux/mutex.h>
+#include <linux/time.h>
+#include <linux/of.h>
+
+#define DBG(fmt...) pr_debug(fmt)
+
+/* see 970FX user manual */
+
+#define SCOM_PCR 0x0aa001 /* PCR scom addr */
+
+#define PCR_HILO_SELECT 0x80000000U /* 1 = PCR, 0 = PCRH */
+#define PCR_SPEED_FULL 0x00000000U /* 1:1 speed value */
+#define PCR_SPEED_HALF 0x00020000U /* 1:2 speed value */
+#define PCR_SPEED_QUARTER 0x00040000U /* 1:4 speed value */
+#define PCR_SPEED_MASK 0x000e0000U /* speed mask */
+#define PCR_SPEED_SHIFT 17
+#define PCR_FREQ_REQ_VALID 0x00010000U /* freq request valid */
+#define PCR_VOLT_REQ_VALID 0x00008000U /* volt request valid */
+#define PCR_TARGET_TIME_MASK 0x00006000U /* target time */
+#define PCR_STATLAT_MASK 0x00001f00U /* STATLAT value */
+#define PCR_SNOOPLAT_MASK 0x000000f0U /* SNOOPLAT value */
+#define PCR_SNOOPACC_MASK 0x0000000fU /* SNOOPACC value */
+
+#define SCOM_PSR 0x408001 /* PSR scom addr */
+/* warning: PSR is a 64 bits register */
+#define PSR_CMD_RECEIVED 0x2000000000000000U /* command received */
+#define PSR_CMD_COMPLETED 0x1000000000000000U /* command completed */
+#define PSR_CUR_SPEED_MASK 0x0300000000000000U /* current speed */
+#define PSR_CUR_SPEED_SHIFT (56)
+
+/*
+ * The G5 only supports two frequencies (Quarter speed is not supported)
+ */
+#define CPUFREQ_HIGH 0
+#define CPUFREQ_LOW 1
+
+static struct cpufreq_frequency_table maple_cpu_freqs[] = {
+ {0, CPUFREQ_HIGH, 0},
+ {0, CPUFREQ_LOW, 0},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+/* Power mode data is an array of the 32 bits PCR values to use for
+ * the various frequencies, retrieved from the device-tree
+ */
+static int maple_pmode_cur;
+
+static const u32 *maple_pmode_data;
+static int maple_pmode_max;
+
+/*
+ * SCOM based frequency switching for 970FX rev3
+ */
+static int maple_scom_switch_freq(int speed_mode)
+{
+ unsigned long flags;
+ int to;
+
+ local_irq_save(flags);
+
+ /* Clear PCR high */
+ scom970_write(SCOM_PCR, 0);
+ /* Clear PCR low */
+ scom970_write(SCOM_PCR, PCR_HILO_SELECT | 0);
+ /* Set PCR low */
+ scom970_write(SCOM_PCR, PCR_HILO_SELECT |
+ maple_pmode_data[speed_mode]);
+
+ /* Wait for completion */
+ for (to = 0; to < 10; to++) {
+ unsigned long psr = scom970_read(SCOM_PSR);
+
+ if ((psr & PSR_CMD_RECEIVED) == 0 &&
+ (((psr >> PSR_CUR_SPEED_SHIFT) ^
+ (maple_pmode_data[speed_mode] >> PCR_SPEED_SHIFT)) & 0x3)
+ == 0)
+ break;
+ if (psr & PSR_CMD_COMPLETED)
+ break;
+ udelay(100);
+ }
+
+ local_irq_restore(flags);
+
+ maple_pmode_cur = speed_mode;
+ ppc_proc_freq = maple_cpu_freqs[speed_mode].frequency * 1000ul;
+
+ return 0;
+}
+
+static int maple_scom_query_freq(void)
+{
+ unsigned long psr = scom970_read(SCOM_PSR);
+ int i;
+
+ for (i = 0; i <= maple_pmode_max; i++)
+ if ((((psr >> PSR_CUR_SPEED_SHIFT) ^
+ (maple_pmode_data[i] >> PCR_SPEED_SHIFT)) & 0x3) == 0)
+ break;
+ return i;
+}
+
+/*
+ * Common interface to the cpufreq core
+ */
+
+static int maple_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ return maple_scom_switch_freq(index);
+}
+
+static unsigned int maple_cpufreq_get_speed(unsigned int cpu)
+{
+ return maple_cpu_freqs[maple_pmode_cur].frequency;
+}
+
+static int maple_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ cpufreq_generic_init(policy, maple_cpu_freqs, 12000);
+ return 0;
+}
+
+static struct cpufreq_driver maple_cpufreq_driver = {
+ .name = "maple",
+ .flags = CPUFREQ_CONST_LOOPS,
+ .init = maple_cpufreq_cpu_init,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = maple_cpufreq_target,
+ .get = maple_cpufreq_get_speed,
+ .attr = cpufreq_generic_attr,
+};
+
+static int __init maple_cpufreq_init(void)
+{
+ struct device_node *cpunode;
+ unsigned int psize;
+ unsigned long max_freq;
+ const u32 *valp;
+ u32 pvr_hi;
+ int rc = -ENODEV;
+
+ /*
+ * Behave here like powermac driver which checks machine compatibility
+ * to ease merging of two drivers in future.
+ */
+ if (!of_machine_is_compatible("Momentum,Maple") &&
+ !of_machine_is_compatible("Momentum,Apache"))
+ return 0;
+
+ /* Get first CPU node */
+ cpunode = of_cpu_device_node_get(0);
+ if (cpunode == NULL) {
+ pr_err("Can't find any CPU 0 node\n");
+ goto bail_noprops;
+ }
+
+ /* Check 970FX for now */
+ /* we actually don't care on which CPU to access PVR */
+ pvr_hi = PVR_VER(mfspr(SPRN_PVR));
+ if (pvr_hi != 0x3c && pvr_hi != 0x44) {
+ pr_err("Unsupported CPU version (%x)\n", pvr_hi);
+ goto bail_noprops;
+ }
+
+ /* Look for the powertune data in the device-tree */
+ /*
+ * On Maple this property is provided by PIBS in dual-processor config,
+ * not provided by PIBS in CPU0 config and also not provided by SLOF,
+ * so YMMV
+ */
+ maple_pmode_data = of_get_property(cpunode, "power-mode-data", &psize);
+ if (!maple_pmode_data) {
+ DBG("No power-mode-data !\n");
+ goto bail_noprops;
+ }
+ maple_pmode_max = psize / sizeof(u32) - 1;
+
+ /*
+ * From what I see, clock-frequency is always the maximal frequency.
+ * The current driver can not slew sysclk yet, so we really only deal
+ * with powertune steps for now. We also only implement full freq and
+ * half freq in this version. So far, I haven't yet seen a machine
+ * supporting anything else.
+ */
+ valp = of_get_property(cpunode, "clock-frequency", NULL);
+ if (!valp)
+ goto bail_noprops;
+ max_freq = (*valp)/1000;
+ maple_cpu_freqs[0].frequency = max_freq;
+ maple_cpu_freqs[1].frequency = max_freq/2;
+
+ /* Force apply current frequency to make sure everything is in
+ * sync (voltage is right for example). Firmware may leave us with
+ * a strange setting ...
+ */
+ msleep(10);
+ maple_pmode_cur = -1;
+ maple_scom_switch_freq(maple_scom_query_freq());
+
+ pr_info("Registering Maple CPU frequency driver\n");
+ pr_info("Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n",
+ maple_cpu_freqs[1].frequency/1000,
+ maple_cpu_freqs[0].frequency/1000,
+ maple_cpu_freqs[maple_pmode_cur].frequency/1000);
+
+ rc = cpufreq_register_driver(&maple_cpufreq_driver);
+
+bail_noprops:
+ of_node_put(cpunode);
+
+ return rc;
+}
+
+module_init(maple_cpufreq_init);
+
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c
new file mode 100644
index 0000000000..d46afb3c00
--- /dev/null
+++ b/drivers/cpufreq/mediatek-cpufreq-hw.c
@@ -0,0 +1,342 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020 MediaTek Inc.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/cpufreq.h>
+#include <linux/energy_model.h>
+#include <linux/init.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define LUT_MAX_ENTRIES 32U
+#define LUT_FREQ GENMASK(11, 0)
+#define LUT_ROW_SIZE 0x4
+#define CPUFREQ_HW_STATUS BIT(0)
+#define SVS_HW_STATUS BIT(1)
+#define POLL_USEC 1000
+#define TIMEOUT_USEC 300000
+
+enum {
+ REG_FREQ_LUT_TABLE,
+ REG_FREQ_ENABLE,
+ REG_FREQ_PERF_STATE,
+ REG_FREQ_HW_STATE,
+ REG_EM_POWER_TBL,
+ REG_FREQ_LATENCY,
+
+ REG_ARRAY_SIZE,
+};
+
+struct mtk_cpufreq_data {
+ struct cpufreq_frequency_table *table;
+ void __iomem *reg_bases[REG_ARRAY_SIZE];
+ struct resource *res;
+ void __iomem *base;
+ int nr_opp;
+};
+
+static const u16 cpufreq_mtk_offsets[REG_ARRAY_SIZE] = {
+ [REG_FREQ_LUT_TABLE] = 0x0,
+ [REG_FREQ_ENABLE] = 0x84,
+ [REG_FREQ_PERF_STATE] = 0x88,
+ [REG_FREQ_HW_STATE] = 0x8c,
+ [REG_EM_POWER_TBL] = 0x90,
+ [REG_FREQ_LATENCY] = 0x110,
+};
+
+static int __maybe_unused
+mtk_cpufreq_get_cpu_power(struct device *cpu_dev, unsigned long *uW,
+ unsigned long *KHz)
+{
+ struct mtk_cpufreq_data *data;
+ struct cpufreq_policy *policy;
+ int i;
+
+ policy = cpufreq_cpu_get_raw(cpu_dev->id);
+ if (!policy)
+ return 0;
+
+ data = policy->driver_data;
+
+ for (i = 0; i < data->nr_opp; i++) {
+ if (data->table[i].frequency < *KHz)
+ break;
+ }
+ i--;
+
+ *KHz = data->table[i].frequency;
+ /* Provide micro-Watts value to the Energy Model */
+ *uW = readl_relaxed(data->reg_bases[REG_EM_POWER_TBL] +
+ i * LUT_ROW_SIZE);
+
+ return 0;
+}
+
+static int mtk_cpufreq_hw_target_index(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct mtk_cpufreq_data *data = policy->driver_data;
+
+ writel_relaxed(index, data->reg_bases[REG_FREQ_PERF_STATE]);
+
+ return 0;
+}
+
+static unsigned int mtk_cpufreq_hw_get(unsigned int cpu)
+{
+ struct mtk_cpufreq_data *data;
+ struct cpufreq_policy *policy;
+ unsigned int index;
+
+ policy = cpufreq_cpu_get_raw(cpu);
+ if (!policy)
+ return 0;
+
+ data = policy->driver_data;
+
+ index = readl_relaxed(data->reg_bases[REG_FREQ_PERF_STATE]);
+ index = min(index, LUT_MAX_ENTRIES - 1);
+
+ return data->table[index].frequency;
+}
+
+static unsigned int mtk_cpufreq_hw_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ struct mtk_cpufreq_data *data = policy->driver_data;
+ unsigned int index;
+
+ index = cpufreq_table_find_index_dl(policy, target_freq, false);
+
+ writel_relaxed(index, data->reg_bases[REG_FREQ_PERF_STATE]);
+
+ return policy->freq_table[index].frequency;
+}
+
+static int mtk_cpu_create_freq_table(struct platform_device *pdev,
+ struct mtk_cpufreq_data *data)
+{
+ struct device *dev = &pdev->dev;
+ u32 temp, i, freq, prev_freq = 0;
+ void __iomem *base_table;
+
+ data->table = devm_kcalloc(dev, LUT_MAX_ENTRIES + 1,
+ sizeof(*data->table), GFP_KERNEL);
+ if (!data->table)
+ return -ENOMEM;
+
+ base_table = data->reg_bases[REG_FREQ_LUT_TABLE];
+
+ for (i = 0; i < LUT_MAX_ENTRIES; i++) {
+ temp = readl_relaxed(base_table + (i * LUT_ROW_SIZE));
+ freq = FIELD_GET(LUT_FREQ, temp) * 1000;
+
+ if (freq == prev_freq)
+ break;
+
+ data->table[i].frequency = freq;
+
+ dev_dbg(dev, "index=%d freq=%d\n", i, data->table[i].frequency);
+
+ prev_freq = freq;
+ }
+
+ data->table[i].frequency = CPUFREQ_TABLE_END;
+ data->nr_opp = i;
+
+ return 0;
+}
+
+static int mtk_cpu_resources_init(struct platform_device *pdev,
+ struct cpufreq_policy *policy,
+ const u16 *offsets)
+{
+ struct mtk_cpufreq_data *data;
+ struct device *dev = &pdev->dev;
+ struct resource *res;
+ struct of_phandle_args args;
+ void __iomem *base;
+ int ret, i;
+ int index;
+
+ data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
+ "#performance-domain-cells",
+ policy->cpus, &args);
+ if (ret < 0)
+ return ret;
+
+ index = args.args[0];
+ of_node_put(args.np);
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, index);
+ if (!res) {
+ dev_err(dev, "failed to get mem resource %d\n", index);
+ return -ENODEV;
+ }
+
+ if (!request_mem_region(res->start, resource_size(res), res->name)) {
+ dev_err(dev, "failed to request resource %pR\n", res);
+ return -EBUSY;
+ }
+
+ base = ioremap(res->start, resource_size(res));
+ if (!base) {
+ dev_err(dev, "failed to map resource %pR\n", res);
+ ret = -ENOMEM;
+ goto release_region;
+ }
+
+ data->base = base;
+ data->res = res;
+
+ for (i = REG_FREQ_LUT_TABLE; i < REG_ARRAY_SIZE; i++)
+ data->reg_bases[i] = base + offsets[i];
+
+ ret = mtk_cpu_create_freq_table(pdev, data);
+ if (ret) {
+ dev_info(dev, "Domain-%d failed to create freq table\n", index);
+ return ret;
+ }
+
+ policy->freq_table = data->table;
+ policy->driver_data = data;
+
+ return 0;
+release_region:
+ release_mem_region(res->start, resource_size(res));
+ return ret;
+}
+
+static int mtk_cpufreq_hw_cpu_init(struct cpufreq_policy *policy)
+{
+ struct platform_device *pdev = cpufreq_get_driver_data();
+ int sig, pwr_hw = CPUFREQ_HW_STATUS | SVS_HW_STATUS;
+ struct mtk_cpufreq_data *data;
+ unsigned int latency;
+ int ret;
+
+ /* Get the bases of cpufreq for domains */
+ ret = mtk_cpu_resources_init(pdev, policy, platform_get_drvdata(pdev));
+ if (ret) {
+ dev_info(&pdev->dev, "CPUFreq resource init failed\n");
+ return ret;
+ }
+
+ data = policy->driver_data;
+
+ latency = readl_relaxed(data->reg_bases[REG_FREQ_LATENCY]) * 1000;
+ if (!latency)
+ latency = CPUFREQ_ETERNAL;
+
+ policy->cpuinfo.transition_latency = latency;
+ policy->fast_switch_possible = true;
+
+ /* HW should be in enabled state to proceed now */
+ writel_relaxed(0x1, data->reg_bases[REG_FREQ_ENABLE]);
+ if (readl_poll_timeout(data->reg_bases[REG_FREQ_HW_STATE], sig,
+ (sig & pwr_hw) == pwr_hw, POLL_USEC,
+ TIMEOUT_USEC)) {
+ if (!(sig & CPUFREQ_HW_STATUS)) {
+ pr_info("cpufreq hardware of CPU%d is not enabled\n",
+ policy->cpu);
+ return -ENODEV;
+ }
+
+ pr_info("SVS of CPU%d is not enabled\n", policy->cpu);
+ }
+
+ return 0;
+}
+
+static int mtk_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct mtk_cpufreq_data *data = policy->driver_data;
+ struct resource *res = data->res;
+ void __iomem *base = data->base;
+
+ /* HW should be in paused state now */
+ writel_relaxed(0x0, data->reg_bases[REG_FREQ_ENABLE]);
+ iounmap(base);
+ release_mem_region(res->start, resource_size(res));
+
+ return 0;
+}
+
+static void mtk_cpufreq_register_em(struct cpufreq_policy *policy)
+{
+ struct em_data_callback em_cb = EM_DATA_CB(mtk_cpufreq_get_cpu_power);
+ struct mtk_cpufreq_data *data = policy->driver_data;
+
+ em_dev_register_perf_domain(get_cpu_device(policy->cpu), data->nr_opp,
+ &em_cb, policy->cpus, true);
+}
+
+static struct cpufreq_driver cpufreq_mtk_hw_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = mtk_cpufreq_hw_target_index,
+ .get = mtk_cpufreq_hw_get,
+ .init = mtk_cpufreq_hw_cpu_init,
+ .exit = mtk_cpufreq_hw_cpu_exit,
+ .register_em = mtk_cpufreq_register_em,
+ .fast_switch = mtk_cpufreq_hw_fast_switch,
+ .name = "mtk-cpufreq-hw",
+ .attr = cpufreq_generic_attr,
+};
+
+static int mtk_cpufreq_hw_driver_probe(struct platform_device *pdev)
+{
+ const void *data;
+ int ret;
+
+ data = of_device_get_match_data(&pdev->dev);
+ if (!data)
+ return -EINVAL;
+
+ platform_set_drvdata(pdev, (void *) data);
+ cpufreq_mtk_hw_driver.driver_data = pdev;
+
+ ret = cpufreq_register_driver(&cpufreq_mtk_hw_driver);
+ if (ret)
+ dev_err(&pdev->dev, "CPUFreq HW driver failed to register\n");
+
+ return ret;
+}
+
+static void mtk_cpufreq_hw_driver_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&cpufreq_mtk_hw_driver);
+}
+
+static const struct of_device_id mtk_cpufreq_hw_match[] = {
+ { .compatible = "mediatek,cpufreq-hw", .data = &cpufreq_mtk_offsets },
+ {}
+};
+MODULE_DEVICE_TABLE(of, mtk_cpufreq_hw_match);
+
+static struct platform_driver mtk_cpufreq_hw_driver = {
+ .probe = mtk_cpufreq_hw_driver_probe,
+ .remove_new = mtk_cpufreq_hw_driver_remove,
+ .driver = {
+ .name = "mtk-cpufreq-hw",
+ .of_match_table = mtk_cpufreq_hw_match,
+ },
+};
+module_platform_driver(mtk_cpufreq_hw_driver);
+
+MODULE_AUTHOR("Hector Yuan <hector.yuan@mediatek.com>");
+MODULE_DESCRIPTION("Mediatek cpufreq-hw driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c
new file mode 100644
index 0000000000..a0a61919bc
--- /dev/null
+++ b/drivers/cpufreq/mediatek-cpufreq.c
@@ -0,0 +1,805 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2015 Linaro Ltd.
+ * Author: Pi-Cheng Chen <pi-cheng.chen@linaro.org>
+ */
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/regulator/consumer.h>
+
+struct mtk_cpufreq_platform_data {
+ int min_volt_shift;
+ int max_volt_shift;
+ int proc_max_volt;
+ int sram_min_volt;
+ int sram_max_volt;
+ bool ccifreq_supported;
+};
+
+/*
+ * The struct mtk_cpu_dvfs_info holds necessary information for doing CPU DVFS
+ * on each CPU power/clock domain of Mediatek SoCs. Each CPU cluster in
+ * Mediatek SoCs has two voltage inputs, Vproc and Vsram. In some cases the two
+ * voltage inputs need to be controlled under a hardware limitation:
+ * 100mV < Vsram - Vproc < 200mV
+ *
+ * When scaling the clock frequency of a CPU clock domain, the clock source
+ * needs to be switched to another stable PLL clock temporarily until
+ * the original PLL becomes stable at target frequency.
+ */
+struct mtk_cpu_dvfs_info {
+ struct cpumask cpus;
+ struct device *cpu_dev;
+ struct device *cci_dev;
+ struct regulator *proc_reg;
+ struct regulator *sram_reg;
+ struct clk *cpu_clk;
+ struct clk *inter_clk;
+ struct list_head list_head;
+ int intermediate_voltage;
+ bool need_voltage_tracking;
+ int vproc_on_boot;
+ int pre_vproc;
+ /* Avoid race condition for regulators between notify and policy */
+ struct mutex reg_lock;
+ struct notifier_block opp_nb;
+ unsigned int opp_cpu;
+ unsigned long current_freq;
+ const struct mtk_cpufreq_platform_data *soc_data;
+ int vtrack_max;
+ bool ccifreq_bound;
+};
+
+static struct platform_device *cpufreq_pdev;
+
+static LIST_HEAD(dvfs_info_list);
+
+static struct mtk_cpu_dvfs_info *mtk_cpu_dvfs_info_lookup(int cpu)
+{
+ struct mtk_cpu_dvfs_info *info;
+
+ list_for_each_entry(info, &dvfs_info_list, list_head) {
+ if (cpumask_test_cpu(cpu, &info->cpus))
+ return info;
+ }
+
+ return NULL;
+}
+
+static int mtk_cpufreq_voltage_tracking(struct mtk_cpu_dvfs_info *info,
+ int new_vproc)
+{
+ const struct mtk_cpufreq_platform_data *soc_data = info->soc_data;
+ struct regulator *proc_reg = info->proc_reg;
+ struct regulator *sram_reg = info->sram_reg;
+ int pre_vproc, pre_vsram, new_vsram, vsram, vproc, ret;
+ int retry = info->vtrack_max;
+
+ pre_vproc = regulator_get_voltage(proc_reg);
+ if (pre_vproc < 0) {
+ dev_err(info->cpu_dev,
+ "invalid Vproc value: %d\n", pre_vproc);
+ return pre_vproc;
+ }
+
+ pre_vsram = regulator_get_voltage(sram_reg);
+ if (pre_vsram < 0) {
+ dev_err(info->cpu_dev, "invalid Vsram value: %d\n", pre_vsram);
+ return pre_vsram;
+ }
+
+ new_vsram = clamp(new_vproc + soc_data->min_volt_shift,
+ soc_data->sram_min_volt, soc_data->sram_max_volt);
+
+ do {
+ if (pre_vproc <= new_vproc) {
+ vsram = clamp(pre_vproc + soc_data->max_volt_shift,
+ soc_data->sram_min_volt, new_vsram);
+ ret = regulator_set_voltage(sram_reg, vsram,
+ soc_data->sram_max_volt);
+
+ if (ret)
+ return ret;
+
+ if (vsram == soc_data->sram_max_volt ||
+ new_vsram == soc_data->sram_min_volt)
+ vproc = new_vproc;
+ else
+ vproc = vsram - soc_data->min_volt_shift;
+
+ ret = regulator_set_voltage(proc_reg, vproc,
+ soc_data->proc_max_volt);
+ if (ret) {
+ regulator_set_voltage(sram_reg, pre_vsram,
+ soc_data->sram_max_volt);
+ return ret;
+ }
+ } else if (pre_vproc > new_vproc) {
+ vproc = max(new_vproc,
+ pre_vsram - soc_data->max_volt_shift);
+ ret = regulator_set_voltage(proc_reg, vproc,
+ soc_data->proc_max_volt);
+ if (ret)
+ return ret;
+
+ if (vproc == new_vproc)
+ vsram = new_vsram;
+ else
+ vsram = max(new_vsram,
+ vproc + soc_data->min_volt_shift);
+
+ ret = regulator_set_voltage(sram_reg, vsram,
+ soc_data->sram_max_volt);
+ if (ret) {
+ regulator_set_voltage(proc_reg, pre_vproc,
+ soc_data->proc_max_volt);
+ return ret;
+ }
+ }
+
+ pre_vproc = vproc;
+ pre_vsram = vsram;
+
+ if (--retry < 0) {
+ dev_err(info->cpu_dev,
+ "over loop count, failed to set voltage\n");
+ return -EINVAL;
+ }
+ } while (vproc != new_vproc || vsram != new_vsram);
+
+ return 0;
+}
+
+static int mtk_cpufreq_set_voltage(struct mtk_cpu_dvfs_info *info, int vproc)
+{
+ const struct mtk_cpufreq_platform_data *soc_data = info->soc_data;
+ int ret;
+
+ if (info->need_voltage_tracking)
+ ret = mtk_cpufreq_voltage_tracking(info, vproc);
+ else
+ ret = regulator_set_voltage(info->proc_reg, vproc,
+ soc_data->proc_max_volt);
+ if (!ret)
+ info->pre_vproc = vproc;
+
+ return ret;
+}
+
+static bool is_ccifreq_ready(struct mtk_cpu_dvfs_info *info)
+{
+ struct device_link *sup_link;
+
+ if (info->ccifreq_bound)
+ return true;
+
+ sup_link = device_link_add(info->cpu_dev, info->cci_dev,
+ DL_FLAG_AUTOREMOVE_CONSUMER);
+ if (!sup_link) {
+ dev_err(info->cpu_dev, "cpu%d: sup_link is NULL\n", info->opp_cpu);
+ return false;
+ }
+
+ if (sup_link->supplier->links.status != DL_DEV_DRIVER_BOUND)
+ return false;
+
+ info->ccifreq_bound = true;
+
+ return true;
+}
+
+static int mtk_cpufreq_set_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct cpufreq_frequency_table *freq_table = policy->freq_table;
+ struct clk *cpu_clk = policy->clk;
+ struct clk *armpll = clk_get_parent(cpu_clk);
+ struct mtk_cpu_dvfs_info *info = policy->driver_data;
+ struct device *cpu_dev = info->cpu_dev;
+ struct dev_pm_opp *opp;
+ long freq_hz, pre_freq_hz;
+ int vproc, pre_vproc, inter_vproc, target_vproc, ret;
+
+ inter_vproc = info->intermediate_voltage;
+
+ pre_freq_hz = clk_get_rate(cpu_clk);
+
+ mutex_lock(&info->reg_lock);
+
+ if (unlikely(info->pre_vproc <= 0))
+ pre_vproc = regulator_get_voltage(info->proc_reg);
+ else
+ pre_vproc = info->pre_vproc;
+
+ if (pre_vproc < 0) {
+ dev_err(cpu_dev, "invalid Vproc value: %d\n", pre_vproc);
+ ret = pre_vproc;
+ goto out;
+ }
+
+ freq_hz = freq_table[index].frequency * 1000;
+
+ opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_hz);
+ if (IS_ERR(opp)) {
+ dev_err(cpu_dev, "cpu%d: failed to find OPP for %ld\n",
+ policy->cpu, freq_hz);
+ ret = PTR_ERR(opp);
+ goto out;
+ }
+ vproc = dev_pm_opp_get_voltage(opp);
+ dev_pm_opp_put(opp);
+
+ /*
+ * If MediaTek cci is supported but is not ready, we will use the value
+ * of max(target cpu voltage, booting voltage) to prevent high freqeuncy
+ * low voltage crash.
+ */
+ if (info->soc_data->ccifreq_supported && !is_ccifreq_ready(info))
+ vproc = max(vproc, info->vproc_on_boot);
+
+ /*
+ * If the new voltage or the intermediate voltage is higher than the
+ * current voltage, scale up voltage first.
+ */
+ target_vproc = max(inter_vproc, vproc);
+ if (pre_vproc <= target_vproc) {
+ ret = mtk_cpufreq_set_voltage(info, target_vproc);
+ if (ret) {
+ dev_err(cpu_dev,
+ "cpu%d: failed to scale up voltage!\n", policy->cpu);
+ mtk_cpufreq_set_voltage(info, pre_vproc);
+ goto out;
+ }
+ }
+
+ /* Reparent the CPU clock to intermediate clock. */
+ ret = clk_set_parent(cpu_clk, info->inter_clk);
+ if (ret) {
+ dev_err(cpu_dev,
+ "cpu%d: failed to re-parent cpu clock!\n", policy->cpu);
+ mtk_cpufreq_set_voltage(info, pre_vproc);
+ goto out;
+ }
+
+ /* Set the original PLL to target rate. */
+ ret = clk_set_rate(armpll, freq_hz);
+ if (ret) {
+ dev_err(cpu_dev,
+ "cpu%d: failed to scale cpu clock rate!\n", policy->cpu);
+ clk_set_parent(cpu_clk, armpll);
+ mtk_cpufreq_set_voltage(info, pre_vproc);
+ goto out;
+ }
+
+ /* Set parent of CPU clock back to the original PLL. */
+ ret = clk_set_parent(cpu_clk, armpll);
+ if (ret) {
+ dev_err(cpu_dev,
+ "cpu%d: failed to re-parent cpu clock!\n", policy->cpu);
+ mtk_cpufreq_set_voltage(info, inter_vproc);
+ goto out;
+ }
+
+ /*
+ * If the new voltage is lower than the intermediate voltage or the
+ * original voltage, scale down to the new voltage.
+ */
+ if (vproc < inter_vproc || vproc < pre_vproc) {
+ ret = mtk_cpufreq_set_voltage(info, vproc);
+ if (ret) {
+ dev_err(cpu_dev,
+ "cpu%d: failed to scale down voltage!\n", policy->cpu);
+ clk_set_parent(cpu_clk, info->inter_clk);
+ clk_set_rate(armpll, pre_freq_hz);
+ clk_set_parent(cpu_clk, armpll);
+ goto out;
+ }
+ }
+
+ info->current_freq = freq_hz;
+
+out:
+ mutex_unlock(&info->reg_lock);
+
+ return ret;
+}
+
+static int mtk_cpufreq_opp_notifier(struct notifier_block *nb,
+ unsigned long event, void *data)
+{
+ struct dev_pm_opp *opp = data;
+ struct dev_pm_opp *new_opp;
+ struct mtk_cpu_dvfs_info *info;
+ unsigned long freq, volt;
+ struct cpufreq_policy *policy;
+ int ret = 0;
+
+ info = container_of(nb, struct mtk_cpu_dvfs_info, opp_nb);
+
+ if (event == OPP_EVENT_ADJUST_VOLTAGE) {
+ freq = dev_pm_opp_get_freq(opp);
+
+ mutex_lock(&info->reg_lock);
+ if (info->current_freq == freq) {
+ volt = dev_pm_opp_get_voltage(opp);
+ ret = mtk_cpufreq_set_voltage(info, volt);
+ if (ret)
+ dev_err(info->cpu_dev,
+ "failed to scale voltage: %d\n", ret);
+ }
+ mutex_unlock(&info->reg_lock);
+ } else if (event == OPP_EVENT_DISABLE) {
+ freq = dev_pm_opp_get_freq(opp);
+
+ /* case of current opp item is disabled */
+ if (info->current_freq == freq) {
+ freq = 1;
+ new_opp = dev_pm_opp_find_freq_ceil(info->cpu_dev,
+ &freq);
+ if (IS_ERR(new_opp)) {
+ dev_err(info->cpu_dev,
+ "all opp items are disabled\n");
+ ret = PTR_ERR(new_opp);
+ return notifier_from_errno(ret);
+ }
+
+ dev_pm_opp_put(new_opp);
+ policy = cpufreq_cpu_get(info->opp_cpu);
+ if (policy) {
+ cpufreq_driver_target(policy, freq / 1000,
+ CPUFREQ_RELATION_L);
+ cpufreq_cpu_put(policy);
+ }
+ }
+ }
+
+ return notifier_from_errno(ret);
+}
+
+static struct device *of_get_cci(struct device *cpu_dev)
+{
+ struct device_node *np;
+ struct platform_device *pdev;
+
+ np = of_parse_phandle(cpu_dev->of_node, "mediatek,cci", 0);
+ if (!np)
+ return ERR_PTR(-ENODEV);
+
+ pdev = of_find_device_by_node(np);
+ of_node_put(np);
+ if (!pdev)
+ return ERR_PTR(-ENODEV);
+
+ return &pdev->dev;
+}
+
+static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
+{
+ struct device *cpu_dev;
+ struct dev_pm_opp *opp;
+ unsigned long rate;
+ int ret;
+
+ cpu_dev = get_cpu_device(cpu);
+ if (!cpu_dev) {
+ dev_err(cpu_dev, "failed to get cpu%d device\n", cpu);
+ return -ENODEV;
+ }
+ info->cpu_dev = cpu_dev;
+
+ info->ccifreq_bound = false;
+ if (info->soc_data->ccifreq_supported) {
+ info->cci_dev = of_get_cci(info->cpu_dev);
+ if (IS_ERR(info->cci_dev)) {
+ ret = PTR_ERR(info->cci_dev);
+ dev_err(cpu_dev, "cpu%d: failed to get cci device\n", cpu);
+ return -ENODEV;
+ }
+ }
+
+ info->cpu_clk = clk_get(cpu_dev, "cpu");
+ if (IS_ERR(info->cpu_clk)) {
+ ret = PTR_ERR(info->cpu_clk);
+ return dev_err_probe(cpu_dev, ret,
+ "cpu%d: failed to get cpu clk\n", cpu);
+ }
+
+ info->inter_clk = clk_get(cpu_dev, "intermediate");
+ if (IS_ERR(info->inter_clk)) {
+ ret = PTR_ERR(info->inter_clk);
+ dev_err_probe(cpu_dev, ret,
+ "cpu%d: failed to get intermediate clk\n", cpu);
+ goto out_free_mux_clock;
+ }
+
+ info->proc_reg = regulator_get_optional(cpu_dev, "proc");
+ if (IS_ERR(info->proc_reg)) {
+ ret = PTR_ERR(info->proc_reg);
+ dev_err_probe(cpu_dev, ret,
+ "cpu%d: failed to get proc regulator\n", cpu);
+ goto out_free_inter_clock;
+ }
+
+ ret = regulator_enable(info->proc_reg);
+ if (ret) {
+ dev_warn(cpu_dev, "cpu%d: failed to enable vproc\n", cpu);
+ goto out_free_proc_reg;
+ }
+
+ /* Both presence and absence of sram regulator are valid cases. */
+ info->sram_reg = regulator_get_optional(cpu_dev, "sram");
+ if (IS_ERR(info->sram_reg)) {
+ ret = PTR_ERR(info->sram_reg);
+ if (ret == -EPROBE_DEFER)
+ goto out_disable_proc_reg;
+
+ info->sram_reg = NULL;
+ } else {
+ ret = regulator_enable(info->sram_reg);
+ if (ret) {
+ dev_warn(cpu_dev, "cpu%d: failed to enable vsram\n", cpu);
+ goto out_free_sram_reg;
+ }
+ }
+
+ /* Get OPP-sharing information from "operating-points-v2" bindings */
+ ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, &info->cpus);
+ if (ret) {
+ dev_err(cpu_dev,
+ "cpu%d: failed to get OPP-sharing information\n", cpu);
+ goto out_disable_sram_reg;
+ }
+
+ ret = dev_pm_opp_of_cpumask_add_table(&info->cpus);
+ if (ret) {
+ dev_warn(cpu_dev, "cpu%d: no OPP table\n", cpu);
+ goto out_disable_sram_reg;
+ }
+
+ ret = clk_prepare_enable(info->cpu_clk);
+ if (ret)
+ goto out_free_opp_table;
+
+ ret = clk_prepare_enable(info->inter_clk);
+ if (ret)
+ goto out_disable_mux_clock;
+
+ if (info->soc_data->ccifreq_supported) {
+ info->vproc_on_boot = regulator_get_voltage(info->proc_reg);
+ if (info->vproc_on_boot < 0) {
+ ret = info->vproc_on_boot;
+ dev_err(info->cpu_dev,
+ "invalid Vproc value: %d\n", info->vproc_on_boot);
+ goto out_disable_inter_clock;
+ }
+ }
+
+ /* Search a safe voltage for intermediate frequency. */
+ rate = clk_get_rate(info->inter_clk);
+ opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate);
+ if (IS_ERR(opp)) {
+ dev_err(cpu_dev, "cpu%d: failed to get intermediate opp\n", cpu);
+ ret = PTR_ERR(opp);
+ goto out_disable_inter_clock;
+ }
+ info->intermediate_voltage = dev_pm_opp_get_voltage(opp);
+ dev_pm_opp_put(opp);
+
+ mutex_init(&info->reg_lock);
+ info->current_freq = clk_get_rate(info->cpu_clk);
+
+ info->opp_cpu = cpu;
+ info->opp_nb.notifier_call = mtk_cpufreq_opp_notifier;
+ ret = dev_pm_opp_register_notifier(cpu_dev, &info->opp_nb);
+ if (ret) {
+ dev_err(cpu_dev, "cpu%d: failed to register opp notifier\n", cpu);
+ goto out_disable_inter_clock;
+ }
+
+ /*
+ * If SRAM regulator is present, software "voltage tracking" is needed
+ * for this CPU power domain.
+ */
+ info->need_voltage_tracking = (info->sram_reg != NULL);
+
+ /*
+ * We assume min voltage is 0 and tracking target voltage using
+ * min_volt_shift for each iteration.
+ * The vtrack_max is 3 times of expeted iteration count.
+ */
+ info->vtrack_max = 3 * DIV_ROUND_UP(max(info->soc_data->sram_max_volt,
+ info->soc_data->proc_max_volt),
+ info->soc_data->min_volt_shift);
+
+ return 0;
+
+out_disable_inter_clock:
+ clk_disable_unprepare(info->inter_clk);
+
+out_disable_mux_clock:
+ clk_disable_unprepare(info->cpu_clk);
+
+out_free_opp_table:
+ dev_pm_opp_of_cpumask_remove_table(&info->cpus);
+
+out_disable_sram_reg:
+ if (info->sram_reg)
+ regulator_disable(info->sram_reg);
+
+out_free_sram_reg:
+ if (info->sram_reg)
+ regulator_put(info->sram_reg);
+
+out_disable_proc_reg:
+ regulator_disable(info->proc_reg);
+
+out_free_proc_reg:
+ regulator_put(info->proc_reg);
+
+out_free_inter_clock:
+ clk_put(info->inter_clk);
+
+out_free_mux_clock:
+ clk_put(info->cpu_clk);
+
+ return ret;
+}
+
+static void mtk_cpu_dvfs_info_release(struct mtk_cpu_dvfs_info *info)
+{
+ regulator_disable(info->proc_reg);
+ regulator_put(info->proc_reg);
+ if (info->sram_reg) {
+ regulator_disable(info->sram_reg);
+ regulator_put(info->sram_reg);
+ }
+ clk_disable_unprepare(info->cpu_clk);
+ clk_put(info->cpu_clk);
+ clk_disable_unprepare(info->inter_clk);
+ clk_put(info->inter_clk);
+ dev_pm_opp_of_cpumask_remove_table(&info->cpus);
+ dev_pm_opp_unregister_notifier(info->cpu_dev, &info->opp_nb);
+}
+
+static int mtk_cpufreq_init(struct cpufreq_policy *policy)
+{
+ struct mtk_cpu_dvfs_info *info;
+ struct cpufreq_frequency_table *freq_table;
+ int ret;
+
+ info = mtk_cpu_dvfs_info_lookup(policy->cpu);
+ if (!info) {
+ pr_err("dvfs info for cpu%d is not initialized.\n",
+ policy->cpu);
+ return -EINVAL;
+ }
+
+ ret = dev_pm_opp_init_cpufreq_table(info->cpu_dev, &freq_table);
+ if (ret) {
+ dev_err(info->cpu_dev,
+ "failed to init cpufreq table for cpu%d: %d\n",
+ policy->cpu, ret);
+ return ret;
+ }
+
+ cpumask_copy(policy->cpus, &info->cpus);
+ policy->freq_table = freq_table;
+ policy->driver_data = info;
+ policy->clk = info->cpu_clk;
+
+ return 0;
+}
+
+static int mtk_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ struct mtk_cpu_dvfs_info *info = policy->driver_data;
+
+ dev_pm_opp_free_cpufreq_table(info->cpu_dev, &policy->freq_table);
+
+ return 0;
+}
+
+static struct cpufreq_driver mtk_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = mtk_cpufreq_set_target,
+ .get = cpufreq_generic_get,
+ .init = mtk_cpufreq_init,
+ .exit = mtk_cpufreq_exit,
+ .register_em = cpufreq_register_em_with_opp,
+ .name = "mtk-cpufreq",
+ .attr = cpufreq_generic_attr,
+};
+
+static int mtk_cpufreq_probe(struct platform_device *pdev)
+{
+ const struct mtk_cpufreq_platform_data *data;
+ struct mtk_cpu_dvfs_info *info, *tmp;
+ int cpu, ret;
+
+ data = dev_get_platdata(&pdev->dev);
+ if (!data) {
+ dev_err(&pdev->dev,
+ "failed to get mtk cpufreq platform data\n");
+ return -ENODEV;
+ }
+
+ for_each_possible_cpu(cpu) {
+ info = mtk_cpu_dvfs_info_lookup(cpu);
+ if (info)
+ continue;
+
+ info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL);
+ if (!info) {
+ ret = -ENOMEM;
+ goto release_dvfs_info_list;
+ }
+
+ info->soc_data = data;
+ ret = mtk_cpu_dvfs_info_init(info, cpu);
+ if (ret) {
+ dev_err(&pdev->dev,
+ "failed to initialize dvfs info for cpu%d\n",
+ cpu);
+ goto release_dvfs_info_list;
+ }
+
+ list_add(&info->list_head, &dvfs_info_list);
+ }
+
+ ret = cpufreq_register_driver(&mtk_cpufreq_driver);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register mtk cpufreq driver\n");
+ goto release_dvfs_info_list;
+ }
+
+ return 0;
+
+release_dvfs_info_list:
+ list_for_each_entry_safe(info, tmp, &dvfs_info_list, list_head) {
+ mtk_cpu_dvfs_info_release(info);
+ list_del(&info->list_head);
+ }
+
+ return ret;
+}
+
+static struct platform_driver mtk_cpufreq_platdrv = {
+ .driver = {
+ .name = "mtk-cpufreq",
+ },
+ .probe = mtk_cpufreq_probe,
+};
+
+static const struct mtk_cpufreq_platform_data mt2701_platform_data = {
+ .min_volt_shift = 100000,
+ .max_volt_shift = 200000,
+ .proc_max_volt = 1150000,
+ .sram_min_volt = 0,
+ .sram_max_volt = 1150000,
+ .ccifreq_supported = false,
+};
+
+static const struct mtk_cpufreq_platform_data mt7622_platform_data = {
+ .min_volt_shift = 100000,
+ .max_volt_shift = 200000,
+ .proc_max_volt = 1350000,
+ .sram_min_volt = 0,
+ .sram_max_volt = 1350000,
+ .ccifreq_supported = false,
+};
+
+static const struct mtk_cpufreq_platform_data mt7623_platform_data = {
+ .min_volt_shift = 100000,
+ .max_volt_shift = 200000,
+ .proc_max_volt = 1300000,
+ .ccifreq_supported = false,
+};
+
+static const struct mtk_cpufreq_platform_data mt8183_platform_data = {
+ .min_volt_shift = 100000,
+ .max_volt_shift = 200000,
+ .proc_max_volt = 1150000,
+ .sram_min_volt = 0,
+ .sram_max_volt = 1150000,
+ .ccifreq_supported = true,
+};
+
+static const struct mtk_cpufreq_platform_data mt8186_platform_data = {
+ .min_volt_shift = 100000,
+ .max_volt_shift = 250000,
+ .proc_max_volt = 1118750,
+ .sram_min_volt = 850000,
+ .sram_max_volt = 1118750,
+ .ccifreq_supported = true,
+};
+
+static const struct mtk_cpufreq_platform_data mt8516_platform_data = {
+ .min_volt_shift = 100000,
+ .max_volt_shift = 200000,
+ .proc_max_volt = 1310000,
+ .sram_min_volt = 0,
+ .sram_max_volt = 1310000,
+ .ccifreq_supported = false,
+};
+
+/* List of machines supported by this driver */
+static const struct of_device_id mtk_cpufreq_machines[] __initconst = {
+ { .compatible = "mediatek,mt2701", .data = &mt2701_platform_data },
+ { .compatible = "mediatek,mt2712", .data = &mt2701_platform_data },
+ { .compatible = "mediatek,mt7622", .data = &mt7622_platform_data },
+ { .compatible = "mediatek,mt7623", .data = &mt7623_platform_data },
+ { .compatible = "mediatek,mt8167", .data = &mt8516_platform_data },
+ { .compatible = "mediatek,mt817x", .data = &mt2701_platform_data },
+ { .compatible = "mediatek,mt8173", .data = &mt2701_platform_data },
+ { .compatible = "mediatek,mt8176", .data = &mt2701_platform_data },
+ { .compatible = "mediatek,mt8183", .data = &mt8183_platform_data },
+ { .compatible = "mediatek,mt8186", .data = &mt8186_platform_data },
+ { .compatible = "mediatek,mt8365", .data = &mt2701_platform_data },
+ { .compatible = "mediatek,mt8516", .data = &mt8516_platform_data },
+ { }
+};
+MODULE_DEVICE_TABLE(of, mtk_cpufreq_machines);
+
+static int __init mtk_cpufreq_driver_init(void)
+{
+ struct device_node *np;
+ const struct of_device_id *match;
+ const struct mtk_cpufreq_platform_data *data;
+ int err;
+
+ np = of_find_node_by_path("/");
+ if (!np)
+ return -ENODEV;
+
+ match = of_match_node(mtk_cpufreq_machines, np);
+ of_node_put(np);
+ if (!match) {
+ pr_debug("Machine is not compatible with mtk-cpufreq\n");
+ return -ENODEV;
+ }
+ data = match->data;
+
+ err = platform_driver_register(&mtk_cpufreq_platdrv);
+ if (err)
+ return err;
+
+ /*
+ * Since there's no place to hold device registration code and no
+ * device tree based way to match cpufreq driver yet, both the driver
+ * and the device registration codes are put here to handle defer
+ * probing.
+ */
+ cpufreq_pdev = platform_device_register_data(NULL, "mtk-cpufreq", -1,
+ data, sizeof(*data));
+ if (IS_ERR(cpufreq_pdev)) {
+ pr_err("failed to register mtk-cpufreq platform device\n");
+ platform_driver_unregister(&mtk_cpufreq_platdrv);
+ return PTR_ERR(cpufreq_pdev);
+ }
+
+ return 0;
+}
+module_init(mtk_cpufreq_driver_init)
+
+static void __exit mtk_cpufreq_driver_exit(void)
+{
+ platform_device_unregister(cpufreq_pdev);
+ platform_driver_unregister(&mtk_cpufreq_platdrv);
+}
+module_exit(mtk_cpufreq_driver_exit)
+
+MODULE_DESCRIPTION("MediaTek CPUFreq driver");
+MODULE_AUTHOR("Pi-Cheng Chen <pi-cheng.chen@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/mvebu-cpufreq.c b/drivers/cpufreq/mvebu-cpufreq.c
new file mode 100644
index 0000000000..7f3cfe668f
--- /dev/null
+++ b/drivers/cpufreq/mvebu-cpufreq.c
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * CPUFreq support for Armada 370/XP platforms.
+ *
+ * Copyright (C) 2012-2016 Marvell
+ *
+ * Yehuda Yitschak <yehuday@marvell.com>
+ * Gregory Clement <gregory.clement@free-electrons.com>
+ * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
+ */
+
+#define pr_fmt(fmt) "mvebu-pmsu: " fmt
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/resource.h>
+
+static int __init armada_xp_pmsu_cpufreq_init(void)
+{
+ struct device_node *np;
+ struct resource res;
+ int ret, cpu;
+
+ if (!of_machine_is_compatible("marvell,armadaxp"))
+ return 0;
+
+ /*
+ * In order to have proper cpufreq handling, we need to ensure
+ * that the Device Tree description of the CPU clock includes
+ * the definition of the PMU DFS registers. If not, we do not
+ * register the clock notifier and the cpufreq driver. This
+ * piece of code is only for compatibility with old Device
+ * Trees.
+ */
+ np = of_find_compatible_node(NULL, NULL, "marvell,armada-xp-cpu-clock");
+ if (!np)
+ return 0;
+
+ ret = of_address_to_resource(np, 1, &res);
+ if (ret) {
+ pr_warn(FW_WARN "not enabling cpufreq, deprecated armada-xp-cpu-clock binding\n");
+ of_node_put(np);
+ return 0;
+ }
+
+ of_node_put(np);
+
+ /*
+ * For each CPU, this loop registers the operating points
+ * supported (which are the nominal CPU frequency and half of
+ * it), and registers the clock notifier that will take care
+ * of doing the PMSU part of a frequency transition.
+ */
+ for_each_possible_cpu(cpu) {
+ struct device *cpu_dev;
+ struct clk *clk;
+ int ret;
+
+ cpu_dev = get_cpu_device(cpu);
+ if (!cpu_dev) {
+ pr_err("Cannot get CPU %d\n", cpu);
+ continue;
+ }
+
+ clk = clk_get(cpu_dev, NULL);
+ if (IS_ERR(clk)) {
+ pr_err("Cannot get clock for CPU %d\n", cpu);
+ return PTR_ERR(clk);
+ }
+
+ ret = dev_pm_opp_add(cpu_dev, clk_get_rate(clk), 0);
+ if (ret) {
+ clk_put(clk);
+ return ret;
+ }
+
+ ret = dev_pm_opp_add(cpu_dev, clk_get_rate(clk) / 2, 0);
+ if (ret) {
+ dev_pm_opp_remove(cpu_dev, clk_get_rate(clk));
+ clk_put(clk);
+ dev_err(cpu_dev, "Failed to register OPPs\n");
+ return ret;
+ }
+
+ ret = dev_pm_opp_set_sharing_cpus(cpu_dev,
+ cpumask_of(cpu_dev->id));
+ if (ret)
+ dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
+ __func__, ret);
+ clk_put(clk);
+ }
+
+ platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+ return 0;
+}
+device_initcall(armada_xp_pmsu_cpufreq_init);
diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c
new file mode 100644
index 0000000000..8956908566
--- /dev/null
+++ b/drivers/cpufreq/omap-cpufreq.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * CPU frequency scaling for OMAP using OPP information
+ *
+ * Copyright (C) 2005 Nokia Corporation
+ * Written by Tony Lindgren <tony@atomide.com>
+ *
+ * Based on cpu-sa1110.c, Copyright (C) 2001 Russell King
+ *
+ * Copyright (C) 2007-2011 Texas Instruments, Inc.
+ * - OMAP3/4 support by Rajendra Nayak, Santosh Shilimkar
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/cpufreq.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/pm_opp.h>
+#include <linux/cpu.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+
+#include <asm/smp_plat.h>
+#include <asm/cpu.h>
+
+/* OPP tolerance in percentage */
+#define OPP_TOLERANCE 4
+
+static struct cpufreq_frequency_table *freq_table;
+static atomic_t freq_table_users = ATOMIC_INIT(0);
+static struct device *mpu_dev;
+static struct regulator *mpu_reg;
+
+static int omap_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ int r, ret;
+ struct dev_pm_opp *opp;
+ unsigned long freq, volt = 0, volt_old = 0, tol = 0;
+ unsigned int old_freq, new_freq;
+
+ old_freq = policy->cur;
+ new_freq = freq_table[index].frequency;
+
+ freq = new_freq * 1000;
+ ret = clk_round_rate(policy->clk, freq);
+ if (ret < 0) {
+ dev_warn(mpu_dev,
+ "CPUfreq: Cannot find matching frequency for %lu\n",
+ freq);
+ return ret;
+ }
+ freq = ret;
+
+ if (mpu_reg) {
+ opp = dev_pm_opp_find_freq_ceil(mpu_dev, &freq);
+ if (IS_ERR(opp)) {
+ dev_err(mpu_dev, "%s: unable to find MPU OPP for %d\n",
+ __func__, new_freq);
+ return -EINVAL;
+ }
+ volt = dev_pm_opp_get_voltage(opp);
+ dev_pm_opp_put(opp);
+ tol = volt * OPP_TOLERANCE / 100;
+ volt_old = regulator_get_voltage(mpu_reg);
+ }
+
+ dev_dbg(mpu_dev, "cpufreq-omap: %u MHz, %ld mV --> %u MHz, %ld mV\n",
+ old_freq / 1000, volt_old ? volt_old / 1000 : -1,
+ new_freq / 1000, volt ? volt / 1000 : -1);
+
+ /* scaling up? scale voltage before frequency */
+ if (mpu_reg && (new_freq > old_freq)) {
+ r = regulator_set_voltage(mpu_reg, volt - tol, volt + tol);
+ if (r < 0) {
+ dev_warn(mpu_dev, "%s: unable to scale voltage up.\n",
+ __func__);
+ return r;
+ }
+ }
+
+ ret = clk_set_rate(policy->clk, new_freq * 1000);
+
+ /* scaling down? scale voltage after frequency */
+ if (mpu_reg && (new_freq < old_freq)) {
+ r = regulator_set_voltage(mpu_reg, volt - tol, volt + tol);
+ if (r < 0) {
+ dev_warn(mpu_dev, "%s: unable to scale voltage down.\n",
+ __func__);
+ clk_set_rate(policy->clk, old_freq * 1000);
+ return r;
+ }
+ }
+
+ return ret;
+}
+
+static inline void freq_table_free(void)
+{
+ if (atomic_dec_and_test(&freq_table_users))
+ dev_pm_opp_free_cpufreq_table(mpu_dev, &freq_table);
+}
+
+static int omap_cpu_init(struct cpufreq_policy *policy)
+{
+ int result;
+
+ policy->clk = clk_get(NULL, "cpufreq_ck");
+ if (IS_ERR(policy->clk))
+ return PTR_ERR(policy->clk);
+
+ if (!freq_table) {
+ result = dev_pm_opp_init_cpufreq_table(mpu_dev, &freq_table);
+ if (result) {
+ dev_err(mpu_dev,
+ "%s: cpu%d: failed creating freq table[%d]\n",
+ __func__, policy->cpu, result);
+ clk_put(policy->clk);
+ return result;
+ }
+ }
+
+ atomic_inc_return(&freq_table_users);
+
+ /* FIXME: what's the actual transition time? */
+ cpufreq_generic_init(policy, freq_table, 300 * 1000);
+
+ return 0;
+}
+
+static int omap_cpu_exit(struct cpufreq_policy *policy)
+{
+ freq_table_free();
+ clk_put(policy->clk);
+ return 0;
+}
+
+static struct cpufreq_driver omap_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = omap_target,
+ .get = cpufreq_generic_get,
+ .init = omap_cpu_init,
+ .exit = omap_cpu_exit,
+ .register_em = cpufreq_register_em_with_opp,
+ .name = "omap",
+ .attr = cpufreq_generic_attr,
+};
+
+static int omap_cpufreq_probe(struct platform_device *pdev)
+{
+ mpu_dev = get_cpu_device(0);
+ if (!mpu_dev) {
+ pr_warn("%s: unable to get the MPU device\n", __func__);
+ return -EINVAL;
+ }
+
+ mpu_reg = regulator_get(mpu_dev, "vcc");
+ if (IS_ERR(mpu_reg)) {
+ pr_warn("%s: unable to get MPU regulator\n", __func__);
+ mpu_reg = NULL;
+ } else {
+ /*
+ * Ensure physical regulator is present.
+ * (e.g. could be dummy regulator.)
+ */
+ if (regulator_get_voltage(mpu_reg) < 0) {
+ pr_warn("%s: physical regulator not present for MPU\n",
+ __func__);
+ regulator_put(mpu_reg);
+ mpu_reg = NULL;
+ }
+ }
+
+ return cpufreq_register_driver(&omap_driver);
+}
+
+static void omap_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&omap_driver);
+}
+
+static struct platform_driver omap_cpufreq_platdrv = {
+ .driver = {
+ .name = "omap-cpufreq",
+ },
+ .probe = omap_cpufreq_probe,
+ .remove_new = omap_cpufreq_remove,
+};
+module_platform_driver(omap_cpufreq_platdrv);
+
+MODULE_DESCRIPTION("cpufreq driver for OMAP SoCs");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/p4-clockmod.c b/drivers/cpufreq/p4-clockmod.c
new file mode 100644
index 0000000000..ef0a3216a3
--- /dev/null
+++ b/drivers/cpufreq/p4-clockmod.c
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Pentium 4/Xeon CPU on demand clock modulation/speed scaling
+ * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ * (C) 2002 Zwane Mwaikambo <zwane@commfireservices.com>
+ * (C) 2002 Arjan van de Ven <arjanv@redhat.com>
+ * (C) 2002 Tora T. Engstad
+ * All Rights Reserved
+ *
+ * The author(s) of this software shall not be held liable for damages
+ * of any nature resulting due to the use of this software. This
+ * software is provided AS-IS with no warranties.
+ *
+ * Date Errata Description
+ * 20020525 N44, O17 12.5% or 25% DC causes lockup
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/timex.h>
+
+#include <asm/processor.h>
+#include <asm/msr.h>
+#include <asm/timer.h>
+#include <asm/cpu_device_id.h>
+
+#include "speedstep-lib.h"
+
+/*
+ * Duty Cycle (3bits), note DC_DISABLE is not specified in
+ * intel docs i just use it to mean disable
+ */
+enum {
+ DC_RESV, DC_DFLT, DC_25PT, DC_38PT, DC_50PT,
+ DC_64PT, DC_75PT, DC_88PT, DC_DISABLE
+};
+
+#define DC_ENTRIES 8
+
+
+static int has_N44_O17_errata[NR_CPUS];
+static unsigned int stock_freq;
+static struct cpufreq_driver p4clockmod_driver;
+static unsigned int cpufreq_p4_get(unsigned int cpu);
+
+static int cpufreq_p4_setdc(unsigned int cpu, unsigned int newstate)
+{
+ u32 l, h;
+
+ if ((newstate > DC_DISABLE) || (newstate == DC_RESV))
+ return -EINVAL;
+
+ rdmsr_on_cpu(cpu, MSR_IA32_THERM_STATUS, &l, &h);
+
+ if (l & 0x01)
+ pr_debug("CPU#%d currently thermal throttled\n", cpu);
+
+ if (has_N44_O17_errata[cpu] &&
+ (newstate == DC_25PT || newstate == DC_DFLT))
+ newstate = DC_38PT;
+
+ rdmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &l, &h);
+ if (newstate == DC_DISABLE) {
+ pr_debug("CPU#%d disabling modulation\n", cpu);
+ wrmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, l & ~(1<<4), h);
+ } else {
+ pr_debug("CPU#%d setting duty cycle to %d%%\n",
+ cpu, ((125 * newstate) / 10));
+ /* bits 63 - 5 : reserved
+ * bit 4 : enable/disable
+ * bits 3-1 : duty cycle
+ * bit 0 : reserved
+ */
+ l = (l & ~14);
+ l = l | (1<<4) | ((newstate & 0x7)<<1);
+ wrmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, l, h);
+ }
+
+ return 0;
+}
+
+
+static struct cpufreq_frequency_table p4clockmod_table[] = {
+ {0, DC_RESV, CPUFREQ_ENTRY_INVALID},
+ {0, DC_DFLT, 0},
+ {0, DC_25PT, 0},
+ {0, DC_38PT, 0},
+ {0, DC_50PT, 0},
+ {0, DC_64PT, 0},
+ {0, DC_75PT, 0},
+ {0, DC_88PT, 0},
+ {0, DC_DISABLE, 0},
+ {0, DC_RESV, CPUFREQ_TABLE_END},
+};
+
+
+static int cpufreq_p4_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ int i;
+
+ /* run on each logical CPU,
+ * see section 13.15.3 of IA32 Intel Architecture Software
+ * Developer's Manual, Volume 3
+ */
+ for_each_cpu(i, policy->cpus)
+ cpufreq_p4_setdc(i, p4clockmod_table[index].driver_data);
+
+ return 0;
+}
+
+
+static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c)
+{
+ if (c->x86 == 0x06) {
+ if (cpu_has(c, X86_FEATURE_EST))
+ pr_warn_once("Warning: EST-capable CPU detected. The acpi-cpufreq module offers voltage scaling in addition to frequency scaling. You should use that instead of p4-clockmod, if possible.\n");
+ switch (c->x86_model) {
+ case 0x0E: /* Core */
+ case 0x0F: /* Core Duo */
+ case 0x16: /* Celeron Core */
+ case 0x1C: /* Atom */
+ p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ return speedstep_get_frequency(SPEEDSTEP_CPU_PCORE);
+ case 0x0D: /* Pentium M (Dothan) */
+ p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ fallthrough;
+ case 0x09: /* Pentium M (Banias) */
+ return speedstep_get_frequency(SPEEDSTEP_CPU_PM);
+ }
+ }
+
+ if (c->x86 != 0xF)
+ return 0;
+
+ /* on P-4s, the TSC runs with constant frequency independent whether
+ * throttling is active or not. */
+ p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+
+ if (speedstep_detect_processor() == SPEEDSTEP_CPU_P4M) {
+ pr_warn("Warning: Pentium 4-M detected. The speedstep-ich or acpi cpufreq modules offer voltage scaling in addition of frequency scaling. You should use either one instead of p4-clockmod, if possible.\n");
+ return speedstep_get_frequency(SPEEDSTEP_CPU_P4M);
+ }
+
+ return speedstep_get_frequency(SPEEDSTEP_CPU_P4D);
+}
+
+
+
+static int cpufreq_p4_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpuinfo_x86 *c = &cpu_data(policy->cpu);
+ int cpuid = 0;
+ unsigned int i;
+
+#ifdef CONFIG_SMP
+ cpumask_copy(policy->cpus, topology_sibling_cpumask(policy->cpu));
+#endif
+
+ /* Errata workaround */
+ cpuid = (c->x86 << 8) | (c->x86_model << 4) | c->x86_stepping;
+ switch (cpuid) {
+ case 0x0f07:
+ case 0x0f0a:
+ case 0x0f11:
+ case 0x0f12:
+ has_N44_O17_errata[policy->cpu] = 1;
+ pr_debug("has errata -- disabling low frequencies\n");
+ }
+
+ if (speedstep_detect_processor() == SPEEDSTEP_CPU_P4D &&
+ c->x86_model < 2) {
+ /* switch to maximum frequency and measure result */
+ cpufreq_p4_setdc(policy->cpu, DC_DISABLE);
+ recalibrate_cpu_khz();
+ }
+ /* get max frequency */
+ stock_freq = cpufreq_p4_get_frequency(c);
+ if (!stock_freq)
+ return -EINVAL;
+
+ /* table init */
+ for (i = 1; (p4clockmod_table[i].frequency != CPUFREQ_TABLE_END); i++) {
+ if ((i < 2) && (has_N44_O17_errata[policy->cpu]))
+ p4clockmod_table[i].frequency = CPUFREQ_ENTRY_INVALID;
+ else
+ p4clockmod_table[i].frequency = (stock_freq * i)/8;
+ }
+
+ /* cpuinfo and default policy values */
+
+ /* the transition latency is set to be 1 higher than the maximum
+ * transition latency of the ondemand governor */
+ policy->cpuinfo.transition_latency = 10000001;
+ policy->freq_table = &p4clockmod_table[0];
+
+ return 0;
+}
+
+
+static unsigned int cpufreq_p4_get(unsigned int cpu)
+{
+ u32 l, h;
+
+ rdmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &l, &h);
+
+ if (l & 0x10) {
+ l = l >> 1;
+ l &= 0x7;
+ } else
+ l = DC_DISABLE;
+
+ if (l != DC_DISABLE)
+ return stock_freq * l / 8;
+
+ return stock_freq;
+}
+
+static struct cpufreq_driver p4clockmod_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = cpufreq_p4_target,
+ .init = cpufreq_p4_cpu_init,
+ .get = cpufreq_p4_get,
+ .name = "p4-clockmod",
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct x86_cpu_id cpufreq_p4_id[] = {
+ X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_ACC, NULL),
+ {}
+};
+
+/*
+ * Intentionally no MODULE_DEVICE_TABLE here: this driver should not
+ * be auto loaded. Please don't add one.
+ */
+
+static int __init cpufreq_p4_init(void)
+{
+ int ret;
+
+ /*
+ * THERM_CONTROL is architectural for IA32 now, so
+ * we can rely on the capability checks
+ */
+ if (!x86_match_cpu(cpufreq_p4_id) || !boot_cpu_has(X86_FEATURE_ACPI))
+ return -ENODEV;
+
+ ret = cpufreq_register_driver(&p4clockmod_driver);
+ if (!ret)
+ pr_info("P4/Xeon(TM) CPU On-Demand Clock Modulation available\n");
+
+ return ret;
+}
+
+
+static void __exit cpufreq_p4_exit(void)
+{
+ cpufreq_unregister_driver(&p4clockmod_driver);
+}
+
+
+MODULE_AUTHOR("Zwane Mwaikambo <zwane@commfireservices.com>");
+MODULE_DESCRIPTION("cpufreq driver for Pentium(TM) 4/Xeon(TM)");
+MODULE_LICENSE("GPL");
+
+late_initcall(cpufreq_p4_init);
+module_exit(cpufreq_p4_exit);
diff --git a/drivers/cpufreq/pasemi-cpufreq.c b/drivers/cpufreq/pasemi-cpufreq.c
new file mode 100644
index 0000000000..039a66bbe1
--- /dev/null
+++ b/drivers/cpufreq/pasemi-cpufreq.c
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2007 PA Semi, Inc
+ *
+ * Authors: Egor Martovetsky <egor@pasemi.com>
+ * Olof Johansson <olof@lixom.net>
+ *
+ * Maintained by: Olof Johansson <olof@lixom.net>
+ *
+ * Based on arch/powerpc/platforms/cell/cbe_cpufreq.c:
+ * (C) Copyright IBM Deutschland Entwicklung GmbH 2005
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/timer.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+
+#include <asm/hw_irq.h>
+#include <asm/io.h>
+#include <asm/time.h>
+#include <asm/smp.h>
+
+#include <platforms/pasemi/pasemi.h>
+
+#define SDCASR_REG 0x0100
+#define SDCASR_REG_STRIDE 0x1000
+#define SDCPWR_CFGA0_REG 0x0100
+#define SDCPWR_PWST0_REG 0x0000
+#define SDCPWR_GIZTIME_REG 0x0440
+
+/* SDCPWR_GIZTIME_REG fields */
+#define SDCPWR_GIZTIME_GR 0x80000000
+#define SDCPWR_GIZTIME_LONGLOCK 0x000000ff
+
+/* Offset of ASR registers from SDC base */
+#define SDCASR_OFFSET 0x120000
+
+static void __iomem *sdcpwr_mapbase;
+static void __iomem *sdcasr_mapbase;
+
+/* Current astate, is used when waking up from power savings on
+ * one core, in case the other core has switched states during
+ * the idle time.
+ */
+static int current_astate;
+
+/* We support 5(A0-A4) power states excluding turbo(A5-A6) modes */
+static struct cpufreq_frequency_table pas_freqs[] = {
+ {0, 0, 0},
+ {0, 1, 0},
+ {0, 2, 0},
+ {0, 3, 0},
+ {0, 4, 0},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+/*
+ * hardware specific functions
+ */
+
+static int get_astate_freq(int astate)
+{
+ u32 ret;
+ ret = in_le32(sdcpwr_mapbase + SDCPWR_CFGA0_REG + (astate * 0x10));
+
+ return ret & 0x3f;
+}
+
+static int get_cur_astate(int cpu)
+{
+ u32 ret;
+
+ ret = in_le32(sdcpwr_mapbase + SDCPWR_PWST0_REG);
+ ret = (ret >> (cpu * 4)) & 0x7;
+
+ return ret;
+}
+
+static int get_gizmo_latency(void)
+{
+ u32 giztime, ret;
+
+ giztime = in_le32(sdcpwr_mapbase + SDCPWR_GIZTIME_REG);
+
+ /* just provide the upper bound */
+ if (giztime & SDCPWR_GIZTIME_GR)
+ ret = (giztime & SDCPWR_GIZTIME_LONGLOCK) * 128000;
+ else
+ ret = (giztime & SDCPWR_GIZTIME_LONGLOCK) * 1000;
+
+ return ret;
+}
+
+static void set_astate(int cpu, unsigned int astate)
+{
+ unsigned long flags;
+
+ /* Return if called before init has run */
+ if (unlikely(!sdcasr_mapbase))
+ return;
+
+ local_irq_save(flags);
+
+ out_le32(sdcasr_mapbase + SDCASR_REG + SDCASR_REG_STRIDE*cpu, astate);
+
+ local_irq_restore(flags);
+}
+
+int check_astate(void)
+{
+ return get_cur_astate(hard_smp_processor_id());
+}
+
+void restore_astate(int cpu)
+{
+ set_astate(cpu, current_astate);
+}
+
+/*
+ * cpufreq functions
+ */
+
+static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *pos;
+ const u32 *max_freqp;
+ u32 max_freq;
+ int cur_astate, idx;
+ struct resource res;
+ struct device_node *cpu, *dn;
+ int err = -ENODEV;
+
+ cpu = of_get_cpu_node(policy->cpu, NULL);
+ if (!cpu)
+ goto out;
+
+ max_freqp = of_get_property(cpu, "clock-frequency", NULL);
+ of_node_put(cpu);
+ if (!max_freqp) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ /* we need the freq in kHz */
+ max_freq = *max_freqp / 1000;
+
+ dn = of_find_compatible_node(NULL, NULL, "1682m-sdc");
+ if (!dn)
+ dn = of_find_compatible_node(NULL, NULL,
+ "pasemi,pwrficient-sdc");
+ if (!dn)
+ goto out;
+ err = of_address_to_resource(dn, 0, &res);
+ of_node_put(dn);
+ if (err)
+ goto out;
+ sdcasr_mapbase = ioremap(res.start + SDCASR_OFFSET, 0x2000);
+ if (!sdcasr_mapbase) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ dn = of_find_compatible_node(NULL, NULL, "1682m-gizmo");
+ if (!dn)
+ dn = of_find_compatible_node(NULL, NULL,
+ "pasemi,pwrficient-gizmo");
+ if (!dn) {
+ err = -ENODEV;
+ goto out_unmap_sdcasr;
+ }
+ err = of_address_to_resource(dn, 0, &res);
+ of_node_put(dn);
+ if (err)
+ goto out_unmap_sdcasr;
+ sdcpwr_mapbase = ioremap(res.start, 0x1000);
+ if (!sdcpwr_mapbase) {
+ err = -EINVAL;
+ goto out_unmap_sdcasr;
+ }
+
+ pr_debug("init cpufreq on CPU %d\n", policy->cpu);
+ pr_debug("max clock-frequency is at %u kHz\n", max_freq);
+ pr_debug("initializing frequency table\n");
+
+ /* initialize frequency table */
+ cpufreq_for_each_entry_idx(pos, pas_freqs, idx) {
+ pos->frequency = get_astate_freq(pos->driver_data) * 100000;
+ pr_debug("%d: %d\n", idx, pos->frequency);
+ }
+
+ cur_astate = get_cur_astate(policy->cpu);
+ pr_debug("current astate is at %d\n",cur_astate);
+
+ policy->cur = pas_freqs[cur_astate].frequency;
+ ppc_proc_freq = policy->cur * 1000ul;
+
+ cpufreq_generic_init(policy, pas_freqs, get_gizmo_latency());
+ return 0;
+
+out_unmap_sdcasr:
+ iounmap(sdcasr_mapbase);
+out:
+ return err;
+}
+
+static int pas_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ /*
+ * We don't support CPU hotplug. Don't unmap after the system
+ * has already made it to a running state.
+ */
+ if (system_state >= SYSTEM_RUNNING)
+ return 0;
+
+ if (sdcasr_mapbase)
+ iounmap(sdcasr_mapbase);
+ if (sdcpwr_mapbase)
+ iounmap(sdcpwr_mapbase);
+
+ return 0;
+}
+
+static int pas_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int pas_astate_new)
+{
+ int i;
+
+ pr_debug("setting frequency for cpu %d to %d kHz, 1/%d of max frequency\n",
+ policy->cpu,
+ pas_freqs[pas_astate_new].frequency,
+ pas_freqs[pas_astate_new].driver_data);
+
+ current_astate = pas_astate_new;
+
+ for_each_online_cpu(i)
+ set_astate(i, pas_astate_new);
+
+ ppc_proc_freq = pas_freqs[pas_astate_new].frequency * 1000ul;
+ return 0;
+}
+
+static struct cpufreq_driver pas_cpufreq_driver = {
+ .name = "pas-cpufreq",
+ .flags = CPUFREQ_CONST_LOOPS,
+ .init = pas_cpufreq_cpu_init,
+ .exit = pas_cpufreq_cpu_exit,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = pas_cpufreq_target,
+ .attr = cpufreq_generic_attr,
+};
+
+/*
+ * module init and destoy
+ */
+
+static int __init pas_cpufreq_init(void)
+{
+ if (!of_machine_is_compatible("PA6T-1682M") &&
+ !of_machine_is_compatible("pasemi,pwrficient"))
+ return -ENODEV;
+
+ return cpufreq_register_driver(&pas_cpufreq_driver);
+}
+
+static void __exit pas_cpufreq_exit(void)
+{
+ cpufreq_unregister_driver(&pas_cpufreq_driver);
+}
+
+module_init(pas_cpufreq_init);
+module_exit(pas_cpufreq_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Egor Martovetsky <egor@pasemi.com>, Olof Johansson <olof@lixom.net>");
diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c
new file mode 100644
index 0000000000..6f8b5ea7ae
--- /dev/null
+++ b/drivers/cpufreq/pcc-cpufreq.c
@@ -0,0 +1,645 @@
+/*
+ * pcc-cpufreq.c - Processor Clocking Control firmware cpufreq interface
+ *
+ * Copyright (C) 2009 Red Hat, Matthew Garrett <mjg@redhat.com>
+ * Copyright (C) 2009 Hewlett-Packard Development Company, L.P.
+ * Nagananda Chumbalkar <nagananda.chumbalkar@hp.com>
+ *
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or NON
+ * INFRINGEMENT. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/cpufreq.h>
+#include <linux/compiler.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+
+#include <linux/acpi.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+
+#include <acpi/processor.h>
+
+#define PCC_VERSION "1.10.00"
+#define POLL_LOOPS 300
+
+#define CMD_COMPLETE 0x1
+#define CMD_GET_FREQ 0x0
+#define CMD_SET_FREQ 0x1
+
+#define BUF_SZ 4
+
+struct pcc_register_resource {
+ u8 descriptor;
+ u16 length;
+ u8 space_id;
+ u8 bit_width;
+ u8 bit_offset;
+ u8 access_size;
+ u64 address;
+} __attribute__ ((packed));
+
+struct pcc_memory_resource {
+ u8 descriptor;
+ u16 length;
+ u8 space_id;
+ u8 resource_usage;
+ u8 type_specific;
+ u64 granularity;
+ u64 minimum;
+ u64 maximum;
+ u64 translation_offset;
+ u64 address_length;
+} __attribute__ ((packed));
+
+static struct cpufreq_driver pcc_cpufreq_driver;
+
+struct pcc_header {
+ u32 signature;
+ u16 length;
+ u8 major;
+ u8 minor;
+ u32 features;
+ u16 command;
+ u16 status;
+ u32 latency;
+ u32 minimum_time;
+ u32 maximum_time;
+ u32 nominal;
+ u32 throttled_frequency;
+ u32 minimum_frequency;
+};
+
+static void __iomem *pcch_virt_addr;
+static struct pcc_header __iomem *pcch_hdr;
+
+static DEFINE_SPINLOCK(pcc_lock);
+
+static struct acpi_generic_address doorbell;
+
+static u64 doorbell_preserve;
+static u64 doorbell_write;
+
+static u8 OSC_UUID[16] = {0x9F, 0x2C, 0x9B, 0x63, 0x91, 0x70, 0x1f, 0x49,
+ 0xBB, 0x4F, 0xA5, 0x98, 0x2F, 0xA1, 0xB5, 0x46};
+
+struct pcc_cpu {
+ u32 input_offset;
+ u32 output_offset;
+};
+
+static struct pcc_cpu __percpu *pcc_cpu_info;
+
+static int pcc_cpufreq_verify(struct cpufreq_policy_data *policy)
+{
+ cpufreq_verify_within_cpu_limits(policy);
+ return 0;
+}
+
+static inline void pcc_cmd(void)
+{
+ u64 doorbell_value;
+ int i;
+
+ acpi_read(&doorbell_value, &doorbell);
+ acpi_write((doorbell_value & doorbell_preserve) | doorbell_write,
+ &doorbell);
+
+ for (i = 0; i < POLL_LOOPS; i++) {
+ if (ioread16(&pcch_hdr->status) & CMD_COMPLETE)
+ break;
+ }
+}
+
+static inline void pcc_clear_mapping(void)
+{
+ if (pcch_virt_addr)
+ iounmap(pcch_virt_addr);
+ pcch_virt_addr = NULL;
+}
+
+static unsigned int pcc_get_freq(unsigned int cpu)
+{
+ struct pcc_cpu *pcc_cpu_data;
+ unsigned int curr_freq;
+ unsigned int freq_limit;
+ u16 status;
+ u32 input_buffer;
+ u32 output_buffer;
+
+ spin_lock(&pcc_lock);
+
+ pr_debug("get: get_freq for CPU %d\n", cpu);
+ pcc_cpu_data = per_cpu_ptr(pcc_cpu_info, cpu);
+
+ input_buffer = 0x1;
+ iowrite32(input_buffer,
+ (pcch_virt_addr + pcc_cpu_data->input_offset));
+ iowrite16(CMD_GET_FREQ, &pcch_hdr->command);
+
+ pcc_cmd();
+
+ output_buffer =
+ ioread32(pcch_virt_addr + pcc_cpu_data->output_offset);
+
+ /* Clear the input buffer - we are done with the current command */
+ memset_io((pcch_virt_addr + pcc_cpu_data->input_offset), 0, BUF_SZ);
+
+ status = ioread16(&pcch_hdr->status);
+ if (status != CMD_COMPLETE) {
+ pr_debug("get: FAILED: for CPU %d, status is %d\n",
+ cpu, status);
+ goto cmd_incomplete;
+ }
+ iowrite16(0, &pcch_hdr->status);
+ curr_freq = (((ioread32(&pcch_hdr->nominal) * (output_buffer & 0xff))
+ / 100) * 1000);
+
+ pr_debug("get: SUCCESS: (virtual) output_offset for cpu %d is "
+ "0x%p, contains a value of: 0x%x. Speed is: %d MHz\n",
+ cpu, (pcch_virt_addr + pcc_cpu_data->output_offset),
+ output_buffer, curr_freq);
+
+ freq_limit = (output_buffer >> 8) & 0xff;
+ if (freq_limit != 0xff) {
+ pr_debug("get: frequency for cpu %d is being temporarily"
+ " capped at %d\n", cpu, curr_freq);
+ }
+
+ spin_unlock(&pcc_lock);
+ return curr_freq;
+
+cmd_incomplete:
+ iowrite16(0, &pcch_hdr->status);
+ spin_unlock(&pcc_lock);
+ return 0;
+}
+
+static int pcc_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ struct pcc_cpu *pcc_cpu_data;
+ struct cpufreq_freqs freqs;
+ u16 status;
+ u32 input_buffer;
+ int cpu;
+
+ cpu = policy->cpu;
+ pcc_cpu_data = per_cpu_ptr(pcc_cpu_info, cpu);
+
+ pr_debug("target: CPU %d should go to target freq: %d "
+ "(virtual) input_offset is 0x%p\n",
+ cpu, target_freq,
+ (pcch_virt_addr + pcc_cpu_data->input_offset));
+
+ freqs.old = policy->cur;
+ freqs.new = target_freq;
+ cpufreq_freq_transition_begin(policy, &freqs);
+ spin_lock(&pcc_lock);
+
+ input_buffer = 0x1 | (((target_freq * 100)
+ / (ioread32(&pcch_hdr->nominal) * 1000)) << 8);
+ iowrite32(input_buffer,
+ (pcch_virt_addr + pcc_cpu_data->input_offset));
+ iowrite16(CMD_SET_FREQ, &pcch_hdr->command);
+
+ pcc_cmd();
+
+ /* Clear the input buffer - we are done with the current command */
+ memset_io((pcch_virt_addr + pcc_cpu_data->input_offset), 0, BUF_SZ);
+
+ status = ioread16(&pcch_hdr->status);
+ iowrite16(0, &pcch_hdr->status);
+
+ spin_unlock(&pcc_lock);
+ cpufreq_freq_transition_end(policy, &freqs, status != CMD_COMPLETE);
+
+ if (status != CMD_COMPLETE) {
+ pr_debug("target: FAILED for cpu %d, with status: 0x%x\n",
+ cpu, status);
+ return -EINVAL;
+ }
+
+ pr_debug("target: was SUCCESSFUL for cpu %d\n", cpu);
+
+ return 0;
+}
+
+static int pcc_get_offset(int cpu)
+{
+ acpi_status status;
+ struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
+ union acpi_object *pccp, *offset;
+ struct pcc_cpu *pcc_cpu_data;
+ struct acpi_processor *pr;
+ int ret = 0;
+
+ pr = per_cpu(processors, cpu);
+ pcc_cpu_data = per_cpu_ptr(pcc_cpu_info, cpu);
+
+ if (!pr)
+ return -ENODEV;
+
+ status = acpi_evaluate_object(pr->handle, "PCCP", NULL, &buffer);
+ if (ACPI_FAILURE(status))
+ return -ENODEV;
+
+ pccp = buffer.pointer;
+ if (!pccp || pccp->type != ACPI_TYPE_PACKAGE) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ offset = &(pccp->package.elements[0]);
+ if (!offset || offset->type != ACPI_TYPE_INTEGER) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ pcc_cpu_data->input_offset = offset->integer.value;
+
+ offset = &(pccp->package.elements[1]);
+ if (!offset || offset->type != ACPI_TYPE_INTEGER) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ pcc_cpu_data->output_offset = offset->integer.value;
+
+ memset_io((pcch_virt_addr + pcc_cpu_data->input_offset), 0, BUF_SZ);
+ memset_io((pcch_virt_addr + pcc_cpu_data->output_offset), 0, BUF_SZ);
+
+ pr_debug("pcc_get_offset: for CPU %d: pcc_cpu_data "
+ "input_offset: 0x%x, pcc_cpu_data output_offset: 0x%x\n",
+ cpu, pcc_cpu_data->input_offset, pcc_cpu_data->output_offset);
+out_free:
+ kfree(buffer.pointer);
+ return ret;
+}
+
+static int __init pcc_cpufreq_do_osc(acpi_handle *handle)
+{
+ acpi_status status;
+ struct acpi_object_list input;
+ struct acpi_buffer output = {ACPI_ALLOCATE_BUFFER, NULL};
+ union acpi_object in_params[4];
+ union acpi_object *out_obj;
+ u32 capabilities[2];
+ u32 errors;
+ u32 supported;
+ int ret = 0;
+
+ input.count = 4;
+ input.pointer = in_params;
+ in_params[0].type = ACPI_TYPE_BUFFER;
+ in_params[0].buffer.length = 16;
+ in_params[0].buffer.pointer = OSC_UUID;
+ in_params[1].type = ACPI_TYPE_INTEGER;
+ in_params[1].integer.value = 1;
+ in_params[2].type = ACPI_TYPE_INTEGER;
+ in_params[2].integer.value = 2;
+ in_params[3].type = ACPI_TYPE_BUFFER;
+ in_params[3].buffer.length = 8;
+ in_params[3].buffer.pointer = (u8 *)&capabilities;
+
+ capabilities[0] = OSC_QUERY_ENABLE;
+ capabilities[1] = 0x1;
+
+ status = acpi_evaluate_object(*handle, "_OSC", &input, &output);
+ if (ACPI_FAILURE(status))
+ return -ENODEV;
+
+ if (!output.length)
+ return -ENODEV;
+
+ out_obj = output.pointer;
+ if (out_obj->type != ACPI_TYPE_BUFFER) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ errors = *((u32 *)out_obj->buffer.pointer) & ~(1 << 0);
+ if (errors) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ supported = *((u32 *)(out_obj->buffer.pointer + 4));
+ if (!(supported & 0x1)) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ kfree(output.pointer);
+ capabilities[0] = 0x0;
+ capabilities[1] = 0x1;
+
+ status = acpi_evaluate_object(*handle, "_OSC", &input, &output);
+ if (ACPI_FAILURE(status))
+ return -ENODEV;
+
+ if (!output.length)
+ return -ENODEV;
+
+ out_obj = output.pointer;
+ if (out_obj->type != ACPI_TYPE_BUFFER) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ errors = *((u32 *)out_obj->buffer.pointer) & ~(1 << 0);
+ if (errors) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ supported = *((u32 *)(out_obj->buffer.pointer + 4));
+ if (!(supported & 0x1)) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+out_free:
+ kfree(output.pointer);
+ return ret;
+}
+
+static int __init pcc_cpufreq_evaluate(void)
+{
+ acpi_status status;
+ struct acpi_buffer output = {ACPI_ALLOCATE_BUFFER, NULL};
+ struct pcc_memory_resource *mem_resource;
+ struct pcc_register_resource *reg_resource;
+ union acpi_object *out_obj, *member;
+ acpi_handle handle, osc_handle;
+ int ret = 0;
+
+ status = acpi_get_handle(NULL, "\\_SB", &handle);
+ if (ACPI_FAILURE(status))
+ return -ENODEV;
+
+ if (!acpi_has_method(handle, "PCCH"))
+ return -ENODEV;
+
+ status = acpi_get_handle(handle, "_OSC", &osc_handle);
+ if (ACPI_SUCCESS(status)) {
+ ret = pcc_cpufreq_do_osc(&osc_handle);
+ if (ret)
+ pr_debug("probe: _OSC evaluation did not succeed\n");
+ /* Firmware's use of _OSC is optional */
+ ret = 0;
+ }
+
+ status = acpi_evaluate_object(handle, "PCCH", NULL, &output);
+ if (ACPI_FAILURE(status))
+ return -ENODEV;
+
+ out_obj = output.pointer;
+ if (out_obj->type != ACPI_TYPE_PACKAGE) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ member = &out_obj->package.elements[0];
+ if (member->type != ACPI_TYPE_BUFFER) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ mem_resource = (struct pcc_memory_resource *)member->buffer.pointer;
+
+ pr_debug("probe: mem_resource descriptor: 0x%x,"
+ " length: %d, space_id: %d, resource_usage: %d,"
+ " type_specific: %d, granularity: 0x%llx,"
+ " minimum: 0x%llx, maximum: 0x%llx,"
+ " translation_offset: 0x%llx, address_length: 0x%llx\n",
+ mem_resource->descriptor, mem_resource->length,
+ mem_resource->space_id, mem_resource->resource_usage,
+ mem_resource->type_specific, mem_resource->granularity,
+ mem_resource->minimum, mem_resource->maximum,
+ mem_resource->translation_offset,
+ mem_resource->address_length);
+
+ if (mem_resource->space_id != ACPI_ADR_SPACE_SYSTEM_MEMORY) {
+ ret = -ENODEV;
+ goto out_free;
+ }
+
+ pcch_virt_addr = ioremap(mem_resource->minimum,
+ mem_resource->address_length);
+ if (pcch_virt_addr == NULL) {
+ pr_debug("probe: could not map shared mem region\n");
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ pcch_hdr = pcch_virt_addr;
+
+ pr_debug("probe: PCCH header (virtual) addr: 0x%p\n", pcch_hdr);
+ pr_debug("probe: PCCH header is at physical address: 0x%llx,"
+ " signature: 0x%x, length: %d bytes, major: %d, minor: %d,"
+ " supported features: 0x%x, command field: 0x%x,"
+ " status field: 0x%x, nominal latency: %d us\n",
+ mem_resource->minimum, ioread32(&pcch_hdr->signature),
+ ioread16(&pcch_hdr->length), ioread8(&pcch_hdr->major),
+ ioread8(&pcch_hdr->minor), ioread32(&pcch_hdr->features),
+ ioread16(&pcch_hdr->command), ioread16(&pcch_hdr->status),
+ ioread32(&pcch_hdr->latency));
+
+ pr_debug("probe: min time between commands: %d us,"
+ " max time between commands: %d us,"
+ " nominal CPU frequency: %d MHz,"
+ " minimum CPU frequency: %d MHz,"
+ " minimum CPU frequency without throttling: %d MHz\n",
+ ioread32(&pcch_hdr->minimum_time),
+ ioread32(&pcch_hdr->maximum_time),
+ ioread32(&pcch_hdr->nominal),
+ ioread32(&pcch_hdr->throttled_frequency),
+ ioread32(&pcch_hdr->minimum_frequency));
+
+ member = &out_obj->package.elements[1];
+ if (member->type != ACPI_TYPE_BUFFER) {
+ ret = -ENODEV;
+ goto pcch_free;
+ }
+
+ reg_resource = (struct pcc_register_resource *)member->buffer.pointer;
+
+ doorbell.space_id = reg_resource->space_id;
+ doorbell.bit_width = reg_resource->bit_width;
+ doorbell.bit_offset = reg_resource->bit_offset;
+ doorbell.access_width = 4;
+ doorbell.address = reg_resource->address;
+
+ pr_debug("probe: doorbell: space_id is %d, bit_width is %d, "
+ "bit_offset is %d, access_width is %d, address is 0x%llx\n",
+ doorbell.space_id, doorbell.bit_width, doorbell.bit_offset,
+ doorbell.access_width, reg_resource->address);
+
+ member = &out_obj->package.elements[2];
+ if (member->type != ACPI_TYPE_INTEGER) {
+ ret = -ENODEV;
+ goto pcch_free;
+ }
+
+ doorbell_preserve = member->integer.value;
+
+ member = &out_obj->package.elements[3];
+ if (member->type != ACPI_TYPE_INTEGER) {
+ ret = -ENODEV;
+ goto pcch_free;
+ }
+
+ doorbell_write = member->integer.value;
+
+ pr_debug("probe: doorbell_preserve: 0x%llx,"
+ " doorbell_write: 0x%llx\n",
+ doorbell_preserve, doorbell_write);
+
+ pcc_cpu_info = alloc_percpu(struct pcc_cpu);
+ if (!pcc_cpu_info) {
+ ret = -ENOMEM;
+ goto pcch_free;
+ }
+
+ printk(KERN_DEBUG "pcc-cpufreq: (v%s) driver loaded with frequency"
+ " limits: %d MHz, %d MHz\n", PCC_VERSION,
+ ioread32(&pcch_hdr->minimum_frequency),
+ ioread32(&pcch_hdr->nominal));
+ kfree(output.pointer);
+ return ret;
+pcch_free:
+ pcc_clear_mapping();
+out_free:
+ kfree(output.pointer);
+ return ret;
+}
+
+static int pcc_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+ unsigned int result = 0;
+
+ if (!pcch_virt_addr) {
+ result = -1;
+ goto out;
+ }
+
+ result = pcc_get_offset(cpu);
+ if (result) {
+ pr_debug("init: PCCP evaluation failed\n");
+ goto out;
+ }
+
+ policy->max = policy->cpuinfo.max_freq =
+ ioread32(&pcch_hdr->nominal) * 1000;
+ policy->min = policy->cpuinfo.min_freq =
+ ioread32(&pcch_hdr->minimum_frequency) * 1000;
+
+ pr_debug("init: policy->max is %d, policy->min is %d\n",
+ policy->max, policy->min);
+out:
+ return result;
+}
+
+static int pcc_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ return 0;
+}
+
+static struct cpufreq_driver pcc_cpufreq_driver = {
+ .flags = CPUFREQ_CONST_LOOPS,
+ .get = pcc_get_freq,
+ .verify = pcc_cpufreq_verify,
+ .target = pcc_cpufreq_target,
+ .init = pcc_cpufreq_cpu_init,
+ .exit = pcc_cpufreq_cpu_exit,
+ .name = "pcc-cpufreq",
+};
+
+static int __init pcc_cpufreq_probe(struct platform_device *pdev)
+{
+ int ret;
+
+ /* Skip initialization if another cpufreq driver is there. */
+ if (cpufreq_get_current_driver())
+ return -ENODEV;
+
+ if (acpi_disabled)
+ return -ENODEV;
+
+ ret = pcc_cpufreq_evaluate();
+ if (ret) {
+ pr_debug("pcc_cpufreq_probe: PCCH evaluation failed\n");
+ return ret;
+ }
+
+ if (num_present_cpus() > 4) {
+ pcc_cpufreq_driver.flags |= CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING;
+ pr_err("%s: Too many CPUs, dynamic performance scaling disabled\n",
+ __func__);
+ pr_err("%s: Try to enable another scaling driver through BIOS settings\n",
+ __func__);
+ pr_err("%s: and complain to the system vendor\n", __func__);
+ }
+
+ ret = cpufreq_register_driver(&pcc_cpufreq_driver);
+
+ return ret;
+}
+
+static void pcc_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&pcc_cpufreq_driver);
+
+ pcc_clear_mapping();
+
+ free_percpu(pcc_cpu_info);
+}
+
+static struct platform_driver pcc_cpufreq_platdrv = {
+ .driver = {
+ .name = "pcc-cpufreq",
+ },
+ .remove_new = pcc_cpufreq_remove,
+};
+
+static int __init pcc_cpufreq_init(void)
+{
+ return platform_driver_probe(&pcc_cpufreq_platdrv, pcc_cpufreq_probe);
+}
+
+static void __exit pcc_cpufreq_exit(void)
+{
+ platform_driver_unregister(&pcc_cpufreq_platdrv);
+}
+
+MODULE_ALIAS("platform:pcc-cpufreq");
+
+MODULE_AUTHOR("Matthew Garrett, Naga Chumbalkar");
+MODULE_VERSION(PCC_VERSION);
+MODULE_DESCRIPTION("Processor Clocking Control interface driver");
+MODULE_LICENSE("GPL");
+
+late_initcall(pcc_cpufreq_init);
+module_exit(pcc_cpufreq_exit);
diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
new file mode 100644
index 0000000000..ec75e79659
--- /dev/null
+++ b/drivers/cpufreq/pmac32-cpufreq.c
@@ -0,0 +1,693 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2002 - 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>
+ * Copyright (C) 2004 John Steele Scott <toojays@toojays.net>
+ *
+ * TODO: Need a big cleanup here. Basically, we need to have different
+ * cpufreq_driver structures for the different type of HW instead of the
+ * current mess. We also need to better deal with the detection of the
+ * type of machine.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+#include <linux/adb.h>
+#include <linux/pmu.h>
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/hardirq.h>
+#include <linux/of.h>
+
+#include <asm/machdep.h>
+#include <asm/irq.h>
+#include <asm/pmac_feature.h>
+#include <asm/mmu_context.h>
+#include <asm/sections.h>
+#include <asm/cputable.h>
+#include <asm/time.h>
+#include <asm/mpic.h>
+#include <asm/keylargo.h>
+#include <asm/switch_to.h>
+
+/* WARNING !!! This will cause calibrate_delay() to be called,
+ * but this is an __init function ! So you MUST go edit
+ * init/main.c to make it non-init before enabling DEBUG_FREQ
+ */
+#undef DEBUG_FREQ
+
+extern void low_choose_7447a_dfs(int dfs);
+extern void low_choose_750fx_pll(int pll);
+extern void low_sleep_handler(void);
+
+/*
+ * Currently, PowerMac cpufreq supports only high & low frequencies
+ * that are set by the firmware
+ */
+static unsigned int low_freq;
+static unsigned int hi_freq;
+static unsigned int cur_freq;
+static unsigned int sleep_freq;
+static unsigned long transition_latency;
+
+/*
+ * Different models uses different mechanisms to switch the frequency
+ */
+static int (*set_speed_proc)(int low_speed);
+static unsigned int (*get_speed_proc)(void);
+
+/*
+ * Some definitions used by the various speedprocs
+ */
+static u32 voltage_gpio;
+static u32 frequency_gpio;
+static u32 slew_done_gpio;
+static int no_schedule;
+static int has_cpu_l2lve;
+static int is_pmu_based;
+
+/* There are only two frequency states for each processor. Values
+ * are in kHz for the time being.
+ */
+#define CPUFREQ_HIGH 0
+#define CPUFREQ_LOW 1
+
+static struct cpufreq_frequency_table pmac_cpu_freqs[] = {
+ {0, CPUFREQ_HIGH, 0},
+ {0, CPUFREQ_LOW, 0},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+static inline void local_delay(unsigned long ms)
+{
+ if (no_schedule)
+ mdelay(ms);
+ else
+ msleep(ms);
+}
+
+#ifdef DEBUG_FREQ
+static inline void debug_calc_bogomips(void)
+{
+ /* This will cause a recalc of bogomips and display the
+ * result. We backup/restore the value to avoid affecting the
+ * core cpufreq framework's own calculation.
+ */
+ unsigned long save_lpj = loops_per_jiffy;
+ calibrate_delay();
+ loops_per_jiffy = save_lpj;
+}
+#endif /* DEBUG_FREQ */
+
+/* Switch CPU speed under 750FX CPU control
+ */
+static int cpu_750fx_cpu_speed(int low_speed)
+{
+ u32 hid2;
+
+ if (low_speed == 0) {
+ /* ramping up, set voltage first */
+ pmac_call_feature(PMAC_FTR_WRITE_GPIO, NULL, voltage_gpio, 0x05);
+ /* Make sure we sleep for at least 1ms */
+ local_delay(10);
+
+ /* tweak L2 for high voltage */
+ if (has_cpu_l2lve) {
+ hid2 = mfspr(SPRN_HID2);
+ hid2 &= ~0x2000;
+ mtspr(SPRN_HID2, hid2);
+ }
+ }
+#ifdef CONFIG_PPC_BOOK3S_32
+ low_choose_750fx_pll(low_speed);
+#endif
+ if (low_speed == 1) {
+ /* tweak L2 for low voltage */
+ if (has_cpu_l2lve) {
+ hid2 = mfspr(SPRN_HID2);
+ hid2 |= 0x2000;
+ mtspr(SPRN_HID2, hid2);
+ }
+
+ /* ramping down, set voltage last */
+ pmac_call_feature(PMAC_FTR_WRITE_GPIO, NULL, voltage_gpio, 0x04);
+ local_delay(10);
+ }
+
+ return 0;
+}
+
+static unsigned int cpu_750fx_get_cpu_speed(void)
+{
+ if (mfspr(SPRN_HID1) & HID1_PS)
+ return low_freq;
+ else
+ return hi_freq;
+}
+
+/* Switch CPU speed using DFS */
+static int dfs_set_cpu_speed(int low_speed)
+{
+ if (low_speed == 0) {
+ /* ramping up, set voltage first */
+ pmac_call_feature(PMAC_FTR_WRITE_GPIO, NULL, voltage_gpio, 0x05);
+ /* Make sure we sleep for at least 1ms */
+ local_delay(1);
+ }
+
+ /* set frequency */
+#ifdef CONFIG_PPC_BOOK3S_32
+ low_choose_7447a_dfs(low_speed);
+#endif
+ udelay(100);
+
+ if (low_speed == 1) {
+ /* ramping down, set voltage last */
+ pmac_call_feature(PMAC_FTR_WRITE_GPIO, NULL, voltage_gpio, 0x04);
+ local_delay(1);
+ }
+
+ return 0;
+}
+
+static unsigned int dfs_get_cpu_speed(void)
+{
+ if (mfspr(SPRN_HID1) & HID1_DFS)
+ return low_freq;
+ else
+ return hi_freq;
+}
+
+
+/* Switch CPU speed using slewing GPIOs
+ */
+static int gpios_set_cpu_speed(int low_speed)
+{
+ int gpio, timeout = 0;
+
+ /* If ramping up, set voltage first */
+ if (low_speed == 0) {
+ pmac_call_feature(PMAC_FTR_WRITE_GPIO, NULL, voltage_gpio, 0x05);
+ /* Delay is way too big but it's ok, we schedule */
+ local_delay(10);
+ }
+
+ /* Set frequency */
+ gpio = pmac_call_feature(PMAC_FTR_READ_GPIO, NULL, frequency_gpio, 0);
+ if (low_speed == ((gpio & 0x01) == 0))
+ goto skip;
+
+ pmac_call_feature(PMAC_FTR_WRITE_GPIO, NULL, frequency_gpio,
+ low_speed ? 0x04 : 0x05);
+ udelay(200);
+ do {
+ if (++timeout > 100)
+ break;
+ local_delay(1);
+ gpio = pmac_call_feature(PMAC_FTR_READ_GPIO, NULL, slew_done_gpio, 0);
+ } while((gpio & 0x02) == 0);
+ skip:
+ /* If ramping down, set voltage last */
+ if (low_speed == 1) {
+ pmac_call_feature(PMAC_FTR_WRITE_GPIO, NULL, voltage_gpio, 0x04);
+ /* Delay is way too big but it's ok, we schedule */
+ local_delay(10);
+ }
+
+#ifdef DEBUG_FREQ
+ debug_calc_bogomips();
+#endif
+
+ return 0;
+}
+
+/* Switch CPU speed under PMU control
+ */
+static int pmu_set_cpu_speed(int low_speed)
+{
+ struct adb_request req;
+ unsigned long save_l2cr;
+ unsigned long save_l3cr;
+ unsigned int pic_prio;
+ unsigned long flags;
+
+ preempt_disable();
+
+#ifdef DEBUG_FREQ
+ printk(KERN_DEBUG "HID1, before: %x\n", mfspr(SPRN_HID1));
+#endif
+ pmu_suspend();
+
+ /* Disable all interrupt sources on openpic */
+ pic_prio = mpic_cpu_get_priority();
+ mpic_cpu_set_priority(0xf);
+
+ /* Make sure the decrementer won't interrupt us */
+ asm volatile("mtdec %0" : : "r" (0x7fffffff));
+ /* Make sure any pending DEC interrupt occurring while we did
+ * the above didn't re-enable the DEC */
+ mb();
+ asm volatile("mtdec %0" : : "r" (0x7fffffff));
+
+ /* We can now disable MSR_EE */
+ local_irq_save(flags);
+
+ /* Giveup the FPU & vec */
+ enable_kernel_fp();
+
+#ifdef CONFIG_ALTIVEC
+ if (cpu_has_feature(CPU_FTR_ALTIVEC))
+ enable_kernel_altivec();
+#endif /* CONFIG_ALTIVEC */
+
+ /* Save & disable L2 and L3 caches */
+ save_l3cr = _get_L3CR(); /* (returns -1 if not available) */
+ save_l2cr = _get_L2CR(); /* (returns -1 if not available) */
+
+ /* Send the new speed command. My assumption is that this command
+ * will cause PLL_CFG[0..3] to be changed next time CPU goes to sleep
+ */
+ pmu_request(&req, NULL, 6, PMU_CPU_SPEED, 'W', 'O', 'O', 'F', low_speed);
+ while (!req.complete)
+ pmu_poll();
+
+ /* Prepare the northbridge for the speed transition */
+ pmac_call_feature(PMAC_FTR_SLEEP_STATE,NULL,1,1);
+
+ /* Call low level code to backup CPU state and recover from
+ * hardware reset
+ */
+ low_sleep_handler();
+
+ /* Restore the northbridge */
+ pmac_call_feature(PMAC_FTR_SLEEP_STATE,NULL,1,0);
+
+ /* Restore L2 cache */
+ if (save_l2cr != 0xffffffff && (save_l2cr & L2CR_L2E) != 0)
+ _set_L2CR(save_l2cr);
+ /* Restore L3 cache */
+ if (save_l3cr != 0xffffffff && (save_l3cr & L3CR_L3E) != 0)
+ _set_L3CR(save_l3cr);
+
+ /* Restore userland MMU context */
+ switch_mmu_context(NULL, current->active_mm, NULL);
+
+#ifdef DEBUG_FREQ
+ printk(KERN_DEBUG "HID1, after: %x\n", mfspr(SPRN_HID1));
+#endif
+
+ /* Restore low level PMU operations */
+ pmu_unlock();
+
+ /*
+ * Restore decrementer; we'll take a decrementer interrupt
+ * as soon as interrupts are re-enabled and the generic
+ * clockevents code will reprogram it with the right value.
+ */
+ set_dec(1);
+
+ /* Restore interrupts */
+ mpic_cpu_set_priority(pic_prio);
+
+ /* Let interrupts flow again ... */
+ local_irq_restore(flags);
+
+#ifdef DEBUG_FREQ
+ debug_calc_bogomips();
+#endif
+
+ pmu_resume();
+
+ preempt_enable();
+
+ return 0;
+}
+
+static int do_set_cpu_speed(struct cpufreq_policy *policy, int speed_mode)
+{
+ unsigned long l3cr;
+ static unsigned long prev_l3cr;
+
+ if (speed_mode == CPUFREQ_LOW &&
+ cpu_has_feature(CPU_FTR_L3CR)) {
+ l3cr = _get_L3CR();
+ if (l3cr & L3CR_L3E) {
+ prev_l3cr = l3cr;
+ _set_L3CR(0);
+ }
+ }
+ set_speed_proc(speed_mode == CPUFREQ_LOW);
+ if (speed_mode == CPUFREQ_HIGH &&
+ cpu_has_feature(CPU_FTR_L3CR)) {
+ l3cr = _get_L3CR();
+ if ((prev_l3cr & L3CR_L3E) && l3cr != prev_l3cr)
+ _set_L3CR(prev_l3cr);
+ }
+ cur_freq = (speed_mode == CPUFREQ_HIGH) ? hi_freq : low_freq;
+
+ return 0;
+}
+
+static unsigned int pmac_cpufreq_get_speed(unsigned int cpu)
+{
+ return cur_freq;
+}
+
+static int pmac_cpufreq_target( struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ int rc;
+
+ rc = do_set_cpu_speed(policy, index);
+
+ ppc_proc_freq = cur_freq * 1000ul;
+ return rc;
+}
+
+static int pmac_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ cpufreq_generic_init(policy, pmac_cpu_freqs, transition_latency);
+ return 0;
+}
+
+static u32 read_gpio(struct device_node *np)
+{
+ const u32 *reg = of_get_property(np, "reg", NULL);
+ u32 offset;
+
+ if (reg == NULL)
+ return 0;
+ /* That works for all keylargos but shall be fixed properly
+ * some day... The problem is that it seems we can't rely
+ * on the "reg" property of the GPIO nodes, they are either
+ * relative to the base of KeyLargo or to the base of the
+ * GPIO space, and the device-tree doesn't help.
+ */
+ offset = *reg;
+ if (offset < KEYLARGO_GPIO_LEVELS0)
+ offset += KEYLARGO_GPIO_LEVELS0;
+ return offset;
+}
+
+static int pmac_cpufreq_suspend(struct cpufreq_policy *policy)
+{
+ /* Ok, this could be made a bit smarter, but let's be robust for now. We
+ * always force a speed change to high speed before sleep, to make sure
+ * we have appropriate voltage and/or bus speed for the wakeup process,
+ * and to make sure our loops_per_jiffies are "good enough", that is will
+ * not cause too short delays if we sleep in low speed and wake in high
+ * speed..
+ */
+ no_schedule = 1;
+ sleep_freq = cur_freq;
+ if (cur_freq == low_freq && !is_pmu_based)
+ do_set_cpu_speed(policy, CPUFREQ_HIGH);
+ return 0;
+}
+
+static int pmac_cpufreq_resume(struct cpufreq_policy *policy)
+{
+ /* If we resume, first check if we have a get() function */
+ if (get_speed_proc)
+ cur_freq = get_speed_proc();
+ else
+ cur_freq = 0;
+
+ /* We don't, hrm... we don't really know our speed here, best
+ * is that we force a switch to whatever it was, which is
+ * probably high speed due to our suspend() routine
+ */
+ do_set_cpu_speed(policy, sleep_freq == low_freq ?
+ CPUFREQ_LOW : CPUFREQ_HIGH);
+
+ ppc_proc_freq = cur_freq * 1000ul;
+
+ no_schedule = 0;
+ return 0;
+}
+
+static struct cpufreq_driver pmac_cpufreq_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = pmac_cpufreq_target,
+ .get = pmac_cpufreq_get_speed,
+ .init = pmac_cpufreq_cpu_init,
+ .suspend = pmac_cpufreq_suspend,
+ .resume = pmac_cpufreq_resume,
+ .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
+ .attr = cpufreq_generic_attr,
+ .name = "powermac",
+};
+
+
+static int pmac_cpufreq_init_MacRISC3(struct device_node *cpunode)
+{
+ struct device_node *volt_gpio_np = of_find_node_by_name(NULL,
+ "voltage-gpio");
+ struct device_node *freq_gpio_np = of_find_node_by_name(NULL,
+ "frequency-gpio");
+ struct device_node *slew_done_gpio_np = of_find_node_by_name(NULL,
+ "slewing-done");
+ const u32 *value;
+
+ /*
+ * Check to see if it's GPIO driven or PMU only
+ *
+ * The way we extract the GPIO address is slightly hackish, but it
+ * works well enough for now. We need to abstract the whole GPIO
+ * stuff sooner or later anyway
+ */
+
+ if (volt_gpio_np)
+ voltage_gpio = read_gpio(volt_gpio_np);
+ if (freq_gpio_np)
+ frequency_gpio = read_gpio(freq_gpio_np);
+ if (slew_done_gpio_np)
+ slew_done_gpio = read_gpio(slew_done_gpio_np);
+
+ of_node_put(volt_gpio_np);
+ of_node_put(freq_gpio_np);
+ of_node_put(slew_done_gpio_np);
+
+ /* If we use the frequency GPIOs, calculate the min/max speeds based
+ * on the bus frequencies
+ */
+ if (frequency_gpio && slew_done_gpio) {
+ int lenp, rc;
+ const u32 *freqs, *ratio;
+
+ freqs = of_get_property(cpunode, "bus-frequencies", &lenp);
+ lenp /= sizeof(u32);
+ if (freqs == NULL || lenp != 2) {
+ pr_err("bus-frequencies incorrect or missing\n");
+ return 1;
+ }
+ ratio = of_get_property(cpunode, "processor-to-bus-ratio*2",
+ NULL);
+ if (ratio == NULL) {
+ pr_err("processor-to-bus-ratio*2 missing\n");
+ return 1;
+ }
+
+ /* Get the min/max bus frequencies */
+ low_freq = min(freqs[0], freqs[1]);
+ hi_freq = max(freqs[0], freqs[1]);
+
+ /* Grrrr.. It _seems_ that the device-tree is lying on the low bus
+ * frequency, it claims it to be around 84Mhz on some models while
+ * it appears to be approx. 101Mhz on all. Let's hack around here...
+ * fortunately, we don't need to be too precise
+ */
+ if (low_freq < 98000000)
+ low_freq = 101000000;
+
+ /* Convert those to CPU core clocks */
+ low_freq = (low_freq * (*ratio)) / 2000;
+ hi_freq = (hi_freq * (*ratio)) / 2000;
+
+ /* Now we get the frequencies, we read the GPIO to see what is out current
+ * speed
+ */
+ rc = pmac_call_feature(PMAC_FTR_READ_GPIO, NULL, frequency_gpio, 0);
+ cur_freq = (rc & 0x01) ? hi_freq : low_freq;
+
+ set_speed_proc = gpios_set_cpu_speed;
+ return 1;
+ }
+
+ /* If we use the PMU, look for the min & max frequencies in the
+ * device-tree
+ */
+ value = of_get_property(cpunode, "min-clock-frequency", NULL);
+ if (!value)
+ return 1;
+ low_freq = (*value) / 1000;
+ /* The PowerBook G4 12" (PowerBook6,1) has an error in the device-tree
+ * here */
+ if (low_freq < 100000)
+ low_freq *= 10;
+
+ value = of_get_property(cpunode, "max-clock-frequency", NULL);
+ if (!value)
+ return 1;
+ hi_freq = (*value) / 1000;
+ set_speed_proc = pmu_set_cpu_speed;
+ is_pmu_based = 1;
+
+ return 0;
+}
+
+static int pmac_cpufreq_init_7447A(struct device_node *cpunode)
+{
+ struct device_node *volt_gpio_np;
+
+ if (!of_property_read_bool(cpunode, "dynamic-power-step"))
+ return 1;
+
+ volt_gpio_np = of_find_node_by_name(NULL, "cpu-vcore-select");
+ if (volt_gpio_np)
+ voltage_gpio = read_gpio(volt_gpio_np);
+ of_node_put(volt_gpio_np);
+ if (!voltage_gpio){
+ pr_err("missing cpu-vcore-select gpio\n");
+ return 1;
+ }
+
+ /* OF only reports the high frequency */
+ hi_freq = cur_freq;
+ low_freq = cur_freq/2;
+
+ /* Read actual frequency from CPU */
+ cur_freq = dfs_get_cpu_speed();
+ set_speed_proc = dfs_set_cpu_speed;
+ get_speed_proc = dfs_get_cpu_speed;
+
+ return 0;
+}
+
+static int pmac_cpufreq_init_750FX(struct device_node *cpunode)
+{
+ struct device_node *volt_gpio_np;
+ u32 pvr;
+ const u32 *value;
+
+ if (!of_property_read_bool(cpunode, "dynamic-power-step"))
+ return 1;
+
+ hi_freq = cur_freq;
+ value = of_get_property(cpunode, "reduced-clock-frequency", NULL);
+ if (!value)
+ return 1;
+ low_freq = (*value) / 1000;
+
+ volt_gpio_np = of_find_node_by_name(NULL, "cpu-vcore-select");
+ if (volt_gpio_np)
+ voltage_gpio = read_gpio(volt_gpio_np);
+
+ of_node_put(volt_gpio_np);
+ pvr = mfspr(SPRN_PVR);
+ has_cpu_l2lve = !((pvr & 0xf00) == 0x100);
+
+ set_speed_proc = cpu_750fx_cpu_speed;
+ get_speed_proc = cpu_750fx_get_cpu_speed;
+ cur_freq = cpu_750fx_get_cpu_speed();
+
+ return 0;
+}
+
+/* Currently, we support the following machines:
+ *
+ * - Titanium PowerBook 1Ghz (PMU based, 667Mhz & 1Ghz)
+ * - Titanium PowerBook 800 (PMU based, 667Mhz & 800Mhz)
+ * - Titanium PowerBook 400 (PMU based, 300Mhz & 400Mhz)
+ * - Titanium PowerBook 500 (PMU based, 300Mhz & 500Mhz)
+ * - iBook2 500/600 (PMU based, 400Mhz & 500/600Mhz)
+ * - iBook2 700 (CPU based, 400Mhz & 700Mhz, support low voltage)
+ * - Recent MacRISC3 laptops
+ * - All new machines with 7447A CPUs
+ */
+static int __init pmac_cpufreq_setup(void)
+{
+ struct device_node *cpunode;
+ const u32 *value;
+
+ if (strstr(boot_command_line, "nocpufreq"))
+ return 0;
+
+ /* Get first CPU node */
+ cpunode = of_cpu_device_node_get(0);
+ if (!cpunode)
+ goto out;
+
+ /* Get current cpu clock freq */
+ value = of_get_property(cpunode, "clock-frequency", NULL);
+ if (!value)
+ goto out;
+ cur_freq = (*value) / 1000;
+
+ /* Check for 7447A based MacRISC3 */
+ if (of_machine_is_compatible("MacRISC3") &&
+ of_property_read_bool(cpunode, "dynamic-power-step") &&
+ PVR_VER(mfspr(SPRN_PVR)) == 0x8003) {
+ pmac_cpufreq_init_7447A(cpunode);
+
+ /* Allow dynamic switching */
+ transition_latency = 8000000;
+ pmac_cpufreq_driver.flags &= ~CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING;
+ /* Check for other MacRISC3 machines */
+ } else if (of_machine_is_compatible("PowerBook3,4") ||
+ of_machine_is_compatible("PowerBook3,5") ||
+ of_machine_is_compatible("MacRISC3")) {
+ pmac_cpufreq_init_MacRISC3(cpunode);
+ /* Else check for iBook2 500/600 */
+ } else if (of_machine_is_compatible("PowerBook4,1")) {
+ hi_freq = cur_freq;
+ low_freq = 400000;
+ set_speed_proc = pmu_set_cpu_speed;
+ is_pmu_based = 1;
+ }
+ /* Else check for TiPb 550 */
+ else if (of_machine_is_compatible("PowerBook3,3") && cur_freq == 550000) {
+ hi_freq = cur_freq;
+ low_freq = 500000;
+ set_speed_proc = pmu_set_cpu_speed;
+ is_pmu_based = 1;
+ }
+ /* Else check for TiPb 400 & 500 */
+ else if (of_machine_is_compatible("PowerBook3,2")) {
+ /* We only know about the 400 MHz and the 500Mhz model
+ * they both have 300 MHz as low frequency
+ */
+ if (cur_freq < 350000 || cur_freq > 550000)
+ goto out;
+ hi_freq = cur_freq;
+ low_freq = 300000;
+ set_speed_proc = pmu_set_cpu_speed;
+ is_pmu_based = 1;
+ }
+ /* Else check for 750FX */
+ else if (PVR_VER(mfspr(SPRN_PVR)) == 0x7000)
+ pmac_cpufreq_init_750FX(cpunode);
+out:
+ of_node_put(cpunode);
+ if (set_speed_proc == NULL)
+ return -ENODEV;
+
+ pmac_cpu_freqs[CPUFREQ_LOW].frequency = low_freq;
+ pmac_cpu_freqs[CPUFREQ_HIGH].frequency = hi_freq;
+ ppc_proc_freq = cur_freq * 1000ul;
+
+ pr_info("Registering PowerMac CPU frequency driver\n");
+ pr_info("Low: %d Mhz, High: %d Mhz, Boot: %d Mhz\n",
+ low_freq/1000, hi_freq/1000, cur_freq/1000);
+
+ return cpufreq_register_driver(&pmac_cpufreq_driver);
+}
+
+module_init(pmac_cpufreq_setup);
+
diff --git a/drivers/cpufreq/pmac64-cpufreq.c b/drivers/cpufreq/pmac64-cpufreq.c
new file mode 100644
index 0000000000..2cd2b06849
--- /dev/null
+++ b/drivers/cpufreq/pmac64-cpufreq.c
@@ -0,0 +1,674 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2002 - 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>
+ * and Markus Demleitner <msdemlei@cl.uni-heidelberg.de>
+ *
+ * This driver adds basic cpufreq support for SMU & 970FX based G5 Macs,
+ * that is iMac G5 and latest single CPU desktop.
+ */
+
+#undef DEBUG
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/completion.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+
+#include <asm/machdep.h>
+#include <asm/irq.h>
+#include <asm/sections.h>
+#include <asm/cputable.h>
+#include <asm/time.h>
+#include <asm/smu.h>
+#include <asm/pmac_pfunc.h>
+
+#define DBG(fmt...) pr_debug(fmt)
+
+/* see 970FX user manual */
+
+#define SCOM_PCR 0x0aa001 /* PCR scom addr */
+
+#define PCR_HILO_SELECT 0x80000000U /* 1 = PCR, 0 = PCRH */
+#define PCR_SPEED_FULL 0x00000000U /* 1:1 speed value */
+#define PCR_SPEED_HALF 0x00020000U /* 1:2 speed value */
+#define PCR_SPEED_QUARTER 0x00040000U /* 1:4 speed value */
+#define PCR_SPEED_MASK 0x000e0000U /* speed mask */
+#define PCR_SPEED_SHIFT 17
+#define PCR_FREQ_REQ_VALID 0x00010000U /* freq request valid */
+#define PCR_VOLT_REQ_VALID 0x00008000U /* volt request valid */
+#define PCR_TARGET_TIME_MASK 0x00006000U /* target time */
+#define PCR_STATLAT_MASK 0x00001f00U /* STATLAT value */
+#define PCR_SNOOPLAT_MASK 0x000000f0U /* SNOOPLAT value */
+#define PCR_SNOOPACC_MASK 0x0000000fU /* SNOOPACC value */
+
+#define SCOM_PSR 0x408001 /* PSR scom addr */
+/* warning: PSR is a 64 bits register */
+#define PSR_CMD_RECEIVED 0x2000000000000000U /* command received */
+#define PSR_CMD_COMPLETED 0x1000000000000000U /* command completed */
+#define PSR_CUR_SPEED_MASK 0x0300000000000000U /* current speed */
+#define PSR_CUR_SPEED_SHIFT (56)
+
+/*
+ * The G5 only supports two frequencies (Quarter speed is not supported)
+ */
+#define CPUFREQ_HIGH 0
+#define CPUFREQ_LOW 1
+
+static struct cpufreq_frequency_table g5_cpu_freqs[] = {
+ {0, CPUFREQ_HIGH, 0},
+ {0, CPUFREQ_LOW, 0},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+/* Power mode data is an array of the 32 bits PCR values to use for
+ * the various frequencies, retrieved from the device-tree
+ */
+static int g5_pmode_cur;
+
+static void (*g5_switch_volt)(int speed_mode);
+static int (*g5_switch_freq)(int speed_mode);
+static int (*g5_query_freq)(void);
+
+static unsigned long transition_latency;
+
+#ifdef CONFIG_PMAC_SMU
+
+static const u32 *g5_pmode_data;
+static int g5_pmode_max;
+
+static struct smu_sdbp_fvt *g5_fvt_table; /* table of op. points */
+static int g5_fvt_count; /* number of op. points */
+static int g5_fvt_cur; /* current op. point */
+
+/*
+ * SMU based voltage switching for Neo2 platforms
+ */
+
+static void g5_smu_switch_volt(int speed_mode)
+{
+ struct smu_simple_cmd cmd;
+
+ DECLARE_COMPLETION_ONSTACK(comp);
+ smu_queue_simple(&cmd, SMU_CMD_POWER_COMMAND, 8, smu_done_complete,
+ &comp, 'V', 'S', 'L', 'E', 'W',
+ 0xff, g5_fvt_cur+1, speed_mode);
+ wait_for_completion(&comp);
+}
+
+/*
+ * Platform function based voltage/vdnap switching for Neo2
+ */
+
+static struct pmf_function *pfunc_set_vdnap0;
+static struct pmf_function *pfunc_vdnap0_complete;
+
+static void g5_vdnap_switch_volt(int speed_mode)
+{
+ struct pmf_args args;
+ u32 slew, done = 0;
+ unsigned long timeout;
+
+ slew = (speed_mode == CPUFREQ_LOW) ? 1 : 0;
+ args.count = 1;
+ args.u[0].p = &slew;
+
+ pmf_call_one(pfunc_set_vdnap0, &args);
+
+ /* It's an irq GPIO so we should be able to just block here,
+ * I'll do that later after I've properly tested the IRQ code for
+ * platform functions
+ */
+ timeout = jiffies + HZ/10;
+ while(!time_after(jiffies, timeout)) {
+ args.count = 1;
+ args.u[0].p = &done;
+ pmf_call_one(pfunc_vdnap0_complete, &args);
+ if (done)
+ break;
+ usleep_range(1000, 1000);
+ }
+ if (done == 0)
+ pr_warn("Timeout in clock slewing !\n");
+}
+
+
+/*
+ * SCOM based frequency switching for 970FX rev3
+ */
+static int g5_scom_switch_freq(int speed_mode)
+{
+ unsigned long flags;
+ int to;
+
+ /* If frequency is going up, first ramp up the voltage */
+ if (speed_mode < g5_pmode_cur)
+ g5_switch_volt(speed_mode);
+
+ local_irq_save(flags);
+
+ /* Clear PCR high */
+ scom970_write(SCOM_PCR, 0);
+ /* Clear PCR low */
+ scom970_write(SCOM_PCR, PCR_HILO_SELECT | 0);
+ /* Set PCR low */
+ scom970_write(SCOM_PCR, PCR_HILO_SELECT |
+ g5_pmode_data[speed_mode]);
+
+ /* Wait for completion */
+ for (to = 0; to < 10; to++) {
+ unsigned long psr = scom970_read(SCOM_PSR);
+
+ if ((psr & PSR_CMD_RECEIVED) == 0 &&
+ (((psr >> PSR_CUR_SPEED_SHIFT) ^
+ (g5_pmode_data[speed_mode] >> PCR_SPEED_SHIFT)) & 0x3)
+ == 0)
+ break;
+ if (psr & PSR_CMD_COMPLETED)
+ break;
+ udelay(100);
+ }
+
+ local_irq_restore(flags);
+
+ /* If frequency is going down, last ramp the voltage */
+ if (speed_mode > g5_pmode_cur)
+ g5_switch_volt(speed_mode);
+
+ g5_pmode_cur = speed_mode;
+ ppc_proc_freq = g5_cpu_freqs[speed_mode].frequency * 1000ul;
+
+ return 0;
+}
+
+static int g5_scom_query_freq(void)
+{
+ unsigned long psr = scom970_read(SCOM_PSR);
+ int i;
+
+ for (i = 0; i <= g5_pmode_max; i++)
+ if ((((psr >> PSR_CUR_SPEED_SHIFT) ^
+ (g5_pmode_data[i] >> PCR_SPEED_SHIFT)) & 0x3) == 0)
+ break;
+ return i;
+}
+
+/*
+ * Fake voltage switching for platforms with missing support
+ */
+
+static void g5_dummy_switch_volt(int speed_mode)
+{
+}
+
+#endif /* CONFIG_PMAC_SMU */
+
+/*
+ * Platform function based voltage switching for PowerMac7,2 & 7,3
+ */
+
+static struct pmf_function *pfunc_cpu0_volt_high;
+static struct pmf_function *pfunc_cpu0_volt_low;
+static struct pmf_function *pfunc_cpu1_volt_high;
+static struct pmf_function *pfunc_cpu1_volt_low;
+
+static void g5_pfunc_switch_volt(int speed_mode)
+{
+ if (speed_mode == CPUFREQ_HIGH) {
+ if (pfunc_cpu0_volt_high)
+ pmf_call_one(pfunc_cpu0_volt_high, NULL);
+ if (pfunc_cpu1_volt_high)
+ pmf_call_one(pfunc_cpu1_volt_high, NULL);
+ } else {
+ if (pfunc_cpu0_volt_low)
+ pmf_call_one(pfunc_cpu0_volt_low, NULL);
+ if (pfunc_cpu1_volt_low)
+ pmf_call_one(pfunc_cpu1_volt_low, NULL);
+ }
+ usleep_range(10000, 10000); /* should be faster , to fix */
+}
+
+/*
+ * Platform function based frequency switching for PowerMac7,2 & 7,3
+ */
+
+static struct pmf_function *pfunc_cpu_setfreq_high;
+static struct pmf_function *pfunc_cpu_setfreq_low;
+static struct pmf_function *pfunc_cpu_getfreq;
+static struct pmf_function *pfunc_slewing_done;
+
+static int g5_pfunc_switch_freq(int speed_mode)
+{
+ struct pmf_args args;
+ u32 done = 0;
+ unsigned long timeout;
+ int rc;
+
+ DBG("g5_pfunc_switch_freq(%d)\n", speed_mode);
+
+ /* If frequency is going up, first ramp up the voltage */
+ if (speed_mode < g5_pmode_cur)
+ g5_switch_volt(speed_mode);
+
+ /* Do it */
+ if (speed_mode == CPUFREQ_HIGH)
+ rc = pmf_call_one(pfunc_cpu_setfreq_high, NULL);
+ else
+ rc = pmf_call_one(pfunc_cpu_setfreq_low, NULL);
+
+ if (rc)
+ pr_warn("pfunc switch error %d\n", rc);
+
+ /* It's an irq GPIO so we should be able to just block here,
+ * I'll do that later after I've properly tested the IRQ code for
+ * platform functions
+ */
+ timeout = jiffies + HZ/10;
+ while(!time_after(jiffies, timeout)) {
+ args.count = 1;
+ args.u[0].p = &done;
+ pmf_call_one(pfunc_slewing_done, &args);
+ if (done)
+ break;
+ usleep_range(500, 500);
+ }
+ if (done == 0)
+ pr_warn("Timeout in clock slewing !\n");
+
+ /* If frequency is going down, last ramp the voltage */
+ if (speed_mode > g5_pmode_cur)
+ g5_switch_volt(speed_mode);
+
+ g5_pmode_cur = speed_mode;
+ ppc_proc_freq = g5_cpu_freqs[speed_mode].frequency * 1000ul;
+
+ return 0;
+}
+
+static int g5_pfunc_query_freq(void)
+{
+ struct pmf_args args;
+ u32 val = 0;
+
+ args.count = 1;
+ args.u[0].p = &val;
+ pmf_call_one(pfunc_cpu_getfreq, &args);
+ return val ? CPUFREQ_HIGH : CPUFREQ_LOW;
+}
+
+
+/*
+ * Common interface to the cpufreq core
+ */
+
+static int g5_cpufreq_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ return g5_switch_freq(index);
+}
+
+static unsigned int g5_cpufreq_get_speed(unsigned int cpu)
+{
+ return g5_cpu_freqs[g5_pmode_cur].frequency;
+}
+
+static int g5_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ cpufreq_generic_init(policy, g5_cpu_freqs, transition_latency);
+ return 0;
+}
+
+static struct cpufreq_driver g5_cpufreq_driver = {
+ .name = "powermac",
+ .flags = CPUFREQ_CONST_LOOPS,
+ .init = g5_cpufreq_cpu_init,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = g5_cpufreq_target,
+ .get = g5_cpufreq_get_speed,
+ .attr = cpufreq_generic_attr,
+};
+
+
+#ifdef CONFIG_PMAC_SMU
+
+static int __init g5_neo2_cpufreq_init(struct device_node *cpunode)
+{
+ unsigned int psize, ssize;
+ unsigned long max_freq;
+ char *freq_method, *volt_method;
+ const u32 *valp;
+ u32 pvr_hi;
+ int use_volts_vdnap = 0;
+ int use_volts_smu = 0;
+ int rc = -ENODEV;
+
+ /* Check supported platforms */
+ if (of_machine_is_compatible("PowerMac8,1") ||
+ of_machine_is_compatible("PowerMac8,2") ||
+ of_machine_is_compatible("PowerMac9,1") ||
+ of_machine_is_compatible("PowerMac12,1"))
+ use_volts_smu = 1;
+ else if (of_machine_is_compatible("PowerMac11,2"))
+ use_volts_vdnap = 1;
+ else
+ return -ENODEV;
+
+ /* Check 970FX for now */
+ valp = of_get_property(cpunode, "cpu-version", NULL);
+ if (!valp) {
+ DBG("No cpu-version property !\n");
+ goto bail_noprops;
+ }
+ pvr_hi = (*valp) >> 16;
+ if (pvr_hi != 0x3c && pvr_hi != 0x44) {
+ pr_err("Unsupported CPU version\n");
+ goto bail_noprops;
+ }
+
+ /* Look for the powertune data in the device-tree */
+ g5_pmode_data = of_get_property(cpunode, "power-mode-data",&psize);
+ if (!g5_pmode_data) {
+ DBG("No power-mode-data !\n");
+ goto bail_noprops;
+ }
+ g5_pmode_max = psize / sizeof(u32) - 1;
+
+ if (use_volts_smu) {
+ const struct smu_sdbp_header *shdr;
+
+ /* Look for the FVT table */
+ shdr = smu_get_sdb_partition(SMU_SDB_FVT_ID, NULL);
+ if (!shdr)
+ goto bail_noprops;
+ g5_fvt_table = (struct smu_sdbp_fvt *)&shdr[1];
+ ssize = (shdr->len * sizeof(u32)) - sizeof(*shdr);
+ g5_fvt_count = ssize / sizeof(*g5_fvt_table);
+ g5_fvt_cur = 0;
+
+ /* Sanity checking */
+ if (g5_fvt_count < 1 || g5_pmode_max < 1)
+ goto bail_noprops;
+
+ g5_switch_volt = g5_smu_switch_volt;
+ volt_method = "SMU";
+ } else if (use_volts_vdnap) {
+ struct device_node *root;
+
+ root = of_find_node_by_path("/");
+ if (root == NULL) {
+ pr_err("Can't find root of device tree\n");
+ goto bail_noprops;
+ }
+ pfunc_set_vdnap0 = pmf_find_function(root, "set-vdnap0");
+ pfunc_vdnap0_complete =
+ pmf_find_function(root, "slewing-done");
+ of_node_put(root);
+ if (pfunc_set_vdnap0 == NULL ||
+ pfunc_vdnap0_complete == NULL) {
+ pr_err("Can't find required platform function\n");
+ goto bail_noprops;
+ }
+
+ g5_switch_volt = g5_vdnap_switch_volt;
+ volt_method = "GPIO";
+ } else {
+ g5_switch_volt = g5_dummy_switch_volt;
+ volt_method = "none";
+ }
+
+ /*
+ * From what I see, clock-frequency is always the maximal frequency.
+ * The current driver can not slew sysclk yet, so we really only deal
+ * with powertune steps for now. We also only implement full freq and
+ * half freq in this version. So far, I haven't yet seen a machine
+ * supporting anything else.
+ */
+ valp = of_get_property(cpunode, "clock-frequency", NULL);
+ if (!valp)
+ return -ENODEV;
+ max_freq = (*valp)/1000;
+ g5_cpu_freqs[0].frequency = max_freq;
+ g5_cpu_freqs[1].frequency = max_freq/2;
+
+ /* Set callbacks */
+ transition_latency = 12000;
+ g5_switch_freq = g5_scom_switch_freq;
+ g5_query_freq = g5_scom_query_freq;
+ freq_method = "SCOM";
+
+ /* Force apply current frequency to make sure everything is in
+ * sync (voltage is right for example). Firmware may leave us with
+ * a strange setting ...
+ */
+ g5_switch_volt(CPUFREQ_HIGH);
+ msleep(10);
+ g5_pmode_cur = -1;
+ g5_switch_freq(g5_query_freq());
+
+ pr_info("Registering G5 CPU frequency driver\n");
+ pr_info("Frequency method: %s, Voltage method: %s\n",
+ freq_method, volt_method);
+ pr_info("Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n",
+ g5_cpu_freqs[1].frequency/1000,
+ g5_cpu_freqs[0].frequency/1000,
+ g5_cpu_freqs[g5_pmode_cur].frequency/1000);
+
+ rc = cpufreq_register_driver(&g5_cpufreq_driver);
+
+ /* We keep the CPU node on hold... hopefully, Apple G5 don't have
+ * hotplug CPU with a dynamic device-tree ...
+ */
+ return rc;
+
+ bail_noprops:
+ of_node_put(cpunode);
+
+ return rc;
+}
+
+#endif /* CONFIG_PMAC_SMU */
+
+
+static int __init g5_pm72_cpufreq_init(struct device_node *cpunode)
+{
+ struct device_node *cpuid = NULL, *hwclock = NULL;
+ const u8 *eeprom = NULL;
+ const u32 *valp;
+ u64 max_freq, min_freq, ih, il;
+ int has_volt = 1, rc = 0;
+
+ DBG("cpufreq: Initializing for PowerMac7,2, PowerMac7,3 and"
+ " RackMac3,1...\n");
+
+ /* Lookup the cpuid eeprom node */
+ cpuid = of_find_node_by_path("/u3@0,f8000000/i2c@f8001000/cpuid@a0");
+ if (cpuid != NULL)
+ eeprom = of_get_property(cpuid, "cpuid", NULL);
+ if (eeprom == NULL) {
+ pr_err("Can't find cpuid EEPROM !\n");
+ rc = -ENODEV;
+ goto bail;
+ }
+
+ /* Lookup the i2c hwclock */
+ for_each_node_by_name(hwclock, "i2c-hwclock") {
+ const char *loc = of_get_property(hwclock,
+ "hwctrl-location", NULL);
+ if (loc == NULL)
+ continue;
+ if (strcmp(loc, "CPU CLOCK"))
+ continue;
+ if (!of_get_property(hwclock, "platform-get-frequency", NULL))
+ continue;
+ break;
+ }
+ if (hwclock == NULL) {
+ pr_err("Can't find i2c clock chip !\n");
+ rc = -ENODEV;
+ goto bail;
+ }
+
+ DBG("cpufreq: i2c clock chip found: %pOF\n", hwclock);
+
+ /* Now get all the platform functions */
+ pfunc_cpu_getfreq =
+ pmf_find_function(hwclock, "get-frequency");
+ pfunc_cpu_setfreq_high =
+ pmf_find_function(hwclock, "set-frequency-high");
+ pfunc_cpu_setfreq_low =
+ pmf_find_function(hwclock, "set-frequency-low");
+ pfunc_slewing_done =
+ pmf_find_function(hwclock, "slewing-done");
+ pfunc_cpu0_volt_high =
+ pmf_find_function(hwclock, "set-voltage-high-0");
+ pfunc_cpu0_volt_low =
+ pmf_find_function(hwclock, "set-voltage-low-0");
+ pfunc_cpu1_volt_high =
+ pmf_find_function(hwclock, "set-voltage-high-1");
+ pfunc_cpu1_volt_low =
+ pmf_find_function(hwclock, "set-voltage-low-1");
+
+ /* Check we have minimum requirements */
+ if (pfunc_cpu_getfreq == NULL || pfunc_cpu_setfreq_high == NULL ||
+ pfunc_cpu_setfreq_low == NULL || pfunc_slewing_done == NULL) {
+ pr_err("Can't find platform functions !\n");
+ rc = -ENODEV;
+ goto bail;
+ }
+
+ /* Check that we have complete sets */
+ if (pfunc_cpu0_volt_high == NULL || pfunc_cpu0_volt_low == NULL) {
+ pmf_put_function(pfunc_cpu0_volt_high);
+ pmf_put_function(pfunc_cpu0_volt_low);
+ pfunc_cpu0_volt_high = pfunc_cpu0_volt_low = NULL;
+ has_volt = 0;
+ }
+ if (!has_volt ||
+ pfunc_cpu1_volt_high == NULL || pfunc_cpu1_volt_low == NULL) {
+ pmf_put_function(pfunc_cpu1_volt_high);
+ pmf_put_function(pfunc_cpu1_volt_low);
+ pfunc_cpu1_volt_high = pfunc_cpu1_volt_low = NULL;
+ }
+
+ /* Note: The device tree also contains a "platform-set-values"
+ * function for which I haven't quite figured out the usage. It
+ * might have to be called on init and/or wakeup, I'm not too sure
+ * but things seem to work fine without it so far ...
+ */
+
+ /* Get max frequency from device-tree */
+ valp = of_get_property(cpunode, "clock-frequency", NULL);
+ if (!valp) {
+ pr_err("Can't find CPU frequency !\n");
+ rc = -ENODEV;
+ goto bail;
+ }
+
+ max_freq = (*valp)/1000;
+
+ /* Now calculate reduced frequency by using the cpuid input freq
+ * ratio. This requires 64 bits math unless we are willing to lose
+ * some precision
+ */
+ ih = *((u32 *)(eeprom + 0x10));
+ il = *((u32 *)(eeprom + 0x20));
+
+ /* Check for machines with no useful settings */
+ if (il == ih) {
+ pr_warn("No low frequency mode available on this model !\n");
+ rc = -ENODEV;
+ goto bail;
+ }
+
+ min_freq = 0;
+ if (ih != 0 && il != 0)
+ min_freq = (max_freq * il) / ih;
+
+ /* Sanity check */
+ if (min_freq >= max_freq || min_freq < 1000) {
+ pr_err("Can't calculate low frequency !\n");
+ rc = -ENXIO;
+ goto bail;
+ }
+ g5_cpu_freqs[0].frequency = max_freq;
+ g5_cpu_freqs[1].frequency = min_freq;
+
+ /* Based on a measurement on Xserve G5, rounded up. */
+ transition_latency = 10 * NSEC_PER_MSEC;
+
+ /* Set callbacks */
+ g5_switch_volt = g5_pfunc_switch_volt;
+ g5_switch_freq = g5_pfunc_switch_freq;
+ g5_query_freq = g5_pfunc_query_freq;
+
+ /* Force apply current frequency to make sure everything is in
+ * sync (voltage is right for example). Firmware may leave us with
+ * a strange setting ...
+ */
+ g5_switch_volt(CPUFREQ_HIGH);
+ msleep(10);
+ g5_pmode_cur = -1;
+ g5_switch_freq(g5_query_freq());
+
+ pr_info("Registering G5 CPU frequency driver\n");
+ pr_info("Frequency method: i2c/pfunc, Voltage method: %s\n",
+ has_volt ? "i2c/pfunc" : "none");
+ pr_info("Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n",
+ g5_cpu_freqs[1].frequency/1000,
+ g5_cpu_freqs[0].frequency/1000,
+ g5_cpu_freqs[g5_pmode_cur].frequency/1000);
+
+ rc = cpufreq_register_driver(&g5_cpufreq_driver);
+ bail:
+ if (rc != 0) {
+ pmf_put_function(pfunc_cpu_getfreq);
+ pmf_put_function(pfunc_cpu_setfreq_high);
+ pmf_put_function(pfunc_cpu_setfreq_low);
+ pmf_put_function(pfunc_slewing_done);
+ pmf_put_function(pfunc_cpu0_volt_high);
+ pmf_put_function(pfunc_cpu0_volt_low);
+ pmf_put_function(pfunc_cpu1_volt_high);
+ pmf_put_function(pfunc_cpu1_volt_low);
+ }
+ of_node_put(hwclock);
+ of_node_put(cpuid);
+ of_node_put(cpunode);
+
+ return rc;
+}
+
+static int __init g5_cpufreq_init(void)
+{
+ struct device_node *cpunode;
+ int rc = 0;
+
+ /* Get first CPU node */
+ cpunode = of_cpu_device_node_get(0);
+ if (cpunode == NULL) {
+ pr_err("Can't find any CPU node\n");
+ return -ENODEV;
+ }
+
+ if (of_machine_is_compatible("PowerMac7,2") ||
+ of_machine_is_compatible("PowerMac7,3") ||
+ of_machine_is_compatible("RackMac3,1"))
+ rc = g5_pm72_cpufreq_init(cpunode);
+#ifdef CONFIG_PMAC_SMU
+ else
+ rc = g5_neo2_cpufreq_init(cpunode);
+#endif /* CONFIG_PMAC_SMU */
+
+ return rc;
+}
+
+module_init(g5_cpufreq_init);
+
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/powernow-k6.c b/drivers/cpufreq/powernow-k6.c
new file mode 100644
index 0000000000..41eefef95d
--- /dev/null
+++ b/drivers/cpufreq/powernow-k6.c
@@ -0,0 +1,311 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This file was based upon code in Powertweak Linux (http://powertweak.sf.net)
+ * (C) 2000-2003 Dave Jones, Arjan van de Ven, Janne Pänkälä,
+ * Dominik Brodowski.
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/ioport.h>
+#include <linux/timex.h>
+#include <linux/io.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/msr.h>
+
+#define POWERNOW_IOPORT 0xfff0 /* it doesn't matter where, as long
+ as it is unused */
+
+static unsigned int busfreq; /* FSB, in 10 kHz */
+static unsigned int max_multiplier;
+
+static unsigned int param_busfreq = 0;
+static unsigned int param_max_multiplier = 0;
+
+module_param_named(max_multiplier, param_max_multiplier, uint, S_IRUGO);
+MODULE_PARM_DESC(max_multiplier, "Maximum multiplier (allowed values: 20 30 35 40 45 50 55 60)");
+
+module_param_named(bus_frequency, param_busfreq, uint, S_IRUGO);
+MODULE_PARM_DESC(bus_frequency, "Bus frequency in kHz");
+
+/* Clock ratio multiplied by 10 - see table 27 in AMD#23446 */
+static struct cpufreq_frequency_table clock_ratio[] = {
+ {0, 60, /* 110 -> 6.0x */ 0},
+ {0, 55, /* 011 -> 5.5x */ 0},
+ {0, 50, /* 001 -> 5.0x */ 0},
+ {0, 45, /* 000 -> 4.5x */ 0},
+ {0, 40, /* 010 -> 4.0x */ 0},
+ {0, 35, /* 111 -> 3.5x */ 0},
+ {0, 30, /* 101 -> 3.0x */ 0},
+ {0, 20, /* 100 -> 2.0x */ 0},
+ {0, 0, CPUFREQ_TABLE_END}
+};
+
+static const u8 index_to_register[8] = { 6, 3, 1, 0, 2, 7, 5, 4 };
+static const u8 register_to_index[8] = { 3, 2, 4, 1, 7, 6, 0, 5 };
+
+static const struct {
+ unsigned freq;
+ unsigned mult;
+} usual_frequency_table[] = {
+ { 350000, 35 }, // 100 * 3.5
+ { 400000, 40 }, // 100 * 4
+ { 450000, 45 }, // 100 * 4.5
+ { 475000, 50 }, // 95 * 5
+ { 500000, 50 }, // 100 * 5
+ { 506250, 45 }, // 112.5 * 4.5
+ { 533500, 55 }, // 97 * 5.5
+ { 550000, 55 }, // 100 * 5.5
+ { 562500, 50 }, // 112.5 * 5
+ { 570000, 60 }, // 95 * 6
+ { 600000, 60 }, // 100 * 6
+ { 618750, 55 }, // 112.5 * 5.5
+ { 660000, 55 }, // 120 * 5.5
+ { 675000, 60 }, // 112.5 * 6
+ { 720000, 60 }, // 120 * 6
+};
+
+#define FREQ_RANGE 3000
+
+/**
+ * powernow_k6_get_cpu_multiplier - returns the current FSB multiplier
+ *
+ * Returns the current setting of the frequency multiplier. Core clock
+ * speed is frequency of the Front-Side Bus multiplied with this value.
+ */
+static int powernow_k6_get_cpu_multiplier(void)
+{
+ unsigned long invalue = 0;
+ u32 msrval;
+
+ local_irq_disable();
+
+ msrval = POWERNOW_IOPORT + 0x1;
+ wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */
+ invalue = inl(POWERNOW_IOPORT + 0x8);
+ msrval = POWERNOW_IOPORT + 0x0;
+ wrmsr(MSR_K6_EPMR, msrval, 0); /* disable it again */
+
+ local_irq_enable();
+
+ return clock_ratio[register_to_index[(invalue >> 5)&7]].driver_data;
+}
+
+static void powernow_k6_set_cpu_multiplier(unsigned int best_i)
+{
+ unsigned long outvalue, invalue;
+ unsigned long msrval;
+ unsigned long cr0;
+
+ /* we now need to transform best_i to the BVC format, see AMD#23446 */
+
+ /*
+ * The processor doesn't respond to inquiry cycles while changing the
+ * frequency, so we must disable cache.
+ */
+ local_irq_disable();
+ cr0 = read_cr0();
+ write_cr0(cr0 | X86_CR0_CD);
+ wbinvd();
+
+ outvalue = (1<<12) | (1<<10) | (1<<9) | (index_to_register[best_i]<<5);
+
+ msrval = POWERNOW_IOPORT + 0x1;
+ wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */
+ invalue = inl(POWERNOW_IOPORT + 0x8);
+ invalue = invalue & 0x1f;
+ outvalue = outvalue | invalue;
+ outl(outvalue, (POWERNOW_IOPORT + 0x8));
+ msrval = POWERNOW_IOPORT + 0x0;
+ wrmsr(MSR_K6_EPMR, msrval, 0); /* disable it again */
+
+ write_cr0(cr0);
+ local_irq_enable();
+}
+
+/**
+ * powernow_k6_target - set the PowerNow! multiplier
+ * @best_i: clock_ratio[best_i] is the target multiplier
+ *
+ * Tries to change the PowerNow! multiplier
+ */
+static int powernow_k6_target(struct cpufreq_policy *policy,
+ unsigned int best_i)
+{
+
+ if (clock_ratio[best_i].driver_data > max_multiplier) {
+ pr_err("invalid target frequency\n");
+ return -EINVAL;
+ }
+
+ powernow_k6_set_cpu_multiplier(best_i);
+
+ return 0;
+}
+
+static int powernow_k6_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *pos;
+ unsigned int i, f;
+ unsigned khz;
+
+ if (policy->cpu != 0)
+ return -ENODEV;
+
+ max_multiplier = 0;
+ khz = cpu_khz;
+ for (i = 0; i < ARRAY_SIZE(usual_frequency_table); i++) {
+ if (khz >= usual_frequency_table[i].freq - FREQ_RANGE &&
+ khz <= usual_frequency_table[i].freq + FREQ_RANGE) {
+ khz = usual_frequency_table[i].freq;
+ max_multiplier = usual_frequency_table[i].mult;
+ break;
+ }
+ }
+ if (param_max_multiplier) {
+ cpufreq_for_each_entry(pos, clock_ratio)
+ if (pos->driver_data == param_max_multiplier) {
+ max_multiplier = param_max_multiplier;
+ goto have_max_multiplier;
+ }
+ pr_err("invalid max_multiplier parameter, valid parameters 20, 30, 35, 40, 45, 50, 55, 60\n");
+ return -EINVAL;
+ }
+
+ if (!max_multiplier) {
+ pr_warn("unknown frequency %u, cannot determine current multiplier\n",
+ khz);
+ pr_warn("use module parameters max_multiplier and bus_frequency\n");
+ return -EOPNOTSUPP;
+ }
+
+have_max_multiplier:
+ param_max_multiplier = max_multiplier;
+
+ if (param_busfreq) {
+ if (param_busfreq >= 50000 && param_busfreq <= 150000) {
+ busfreq = param_busfreq / 10;
+ goto have_busfreq;
+ }
+ pr_err("invalid bus_frequency parameter, allowed range 50000 - 150000 kHz\n");
+ return -EINVAL;
+ }
+
+ busfreq = khz / max_multiplier;
+have_busfreq:
+ param_busfreq = busfreq * 10;
+
+ /* table init */
+ cpufreq_for_each_entry(pos, clock_ratio) {
+ f = pos->driver_data;
+ if (f > max_multiplier)
+ pos->frequency = CPUFREQ_ENTRY_INVALID;
+ else
+ pos->frequency = busfreq * f;
+ }
+
+ /* cpuinfo and default policy values */
+ policy->cpuinfo.transition_latency = 500000;
+ policy->freq_table = clock_ratio;
+
+ return 0;
+}
+
+
+static int powernow_k6_cpu_exit(struct cpufreq_policy *policy)
+{
+ unsigned int i;
+
+ for (i = 0; (clock_ratio[i].frequency != CPUFREQ_TABLE_END); i++) {
+ if (clock_ratio[i].driver_data == max_multiplier) {
+ struct cpufreq_freqs freqs;
+
+ freqs.old = policy->cur;
+ freqs.new = clock_ratio[i].frequency;
+ freqs.flags = 0;
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+ powernow_k6_target(policy, i);
+ cpufreq_freq_transition_end(policy, &freqs, 0);
+ break;
+ }
+ }
+ return 0;
+}
+
+static unsigned int powernow_k6_get(unsigned int cpu)
+{
+ unsigned int ret;
+ ret = (busfreq * powernow_k6_get_cpu_multiplier());
+ return ret;
+}
+
+static struct cpufreq_driver powernow_k6_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = powernow_k6_target,
+ .init = powernow_k6_cpu_init,
+ .exit = powernow_k6_cpu_exit,
+ .get = powernow_k6_get,
+ .name = "powernow-k6",
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct x86_cpu_id powernow_k6_ids[] = {
+ X86_MATCH_VENDOR_FAM_MODEL(AMD, 5, 12, NULL),
+ X86_MATCH_VENDOR_FAM_MODEL(AMD, 5, 13, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, powernow_k6_ids);
+
+/**
+ * powernow_k6_init - initializes the k6 PowerNow! CPUFreq driver
+ *
+ * Initializes the K6 PowerNow! support. Returns -ENODEV on unsupported
+ * devices, -EINVAL or -ENOMEM on problems during initiatization, and zero
+ * on success.
+ */
+static int __init powernow_k6_init(void)
+{
+ if (!x86_match_cpu(powernow_k6_ids))
+ return -ENODEV;
+
+ if (!request_region(POWERNOW_IOPORT, 16, "PowerNow!")) {
+ pr_info("PowerNow IOPORT region already used\n");
+ return -EIO;
+ }
+
+ if (cpufreq_register_driver(&powernow_k6_driver)) {
+ release_region(POWERNOW_IOPORT, 16);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+
+/**
+ * powernow_k6_exit - unregisters AMD K6-2+/3+ PowerNow! support
+ *
+ * Unregisters AMD K6-2+ / K6-3+ PowerNow! support.
+ */
+static void __exit powernow_k6_exit(void)
+{
+ cpufreq_unregister_driver(&powernow_k6_driver);
+ release_region(POWERNOW_IOPORT, 16);
+}
+
+
+MODULE_AUTHOR("Arjan van de Ven, Dave Jones, "
+ "Dominik Brodowski <linux@brodo.de>");
+MODULE_DESCRIPTION("PowerNow! driver for AMD K6-2+ / K6-3+ processors.");
+MODULE_LICENSE("GPL");
+
+module_init(powernow_k6_init);
+module_exit(powernow_k6_exit);
diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c
new file mode 100644
index 0000000000..5d515fc348
--- /dev/null
+++ b/drivers/cpufreq/powernow-k7.c
@@ -0,0 +1,696 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD K7 Powernow driver.
+ * (C) 2003 Dave Jones on behalf of SuSE Labs.
+ *
+ * Based upon datasheets & sample CPUs kindly provided by AMD.
+ *
+ * Errata 5:
+ * CPU may fail to execute a FID/VID change in presence of interrupt.
+ * - We cli/sti on stepping A0 CPUs around the FID/VID transition.
+ * Errata 15:
+ * CPU with half frequency multipliers may hang upon wakeup from disconnect.
+ * - We disable half multipliers if ACPI is used on A0 stepping CPUs.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/dmi.h>
+#include <linux/timex.h>
+#include <linux/io.h>
+
+#include <asm/timer.h> /* Needed for recalibrate_cpu_khz() */
+#include <asm/msr.h>
+#include <asm/cpu_device_id.h>
+
+#ifdef CONFIG_X86_POWERNOW_K7_ACPI
+#include <linux/acpi.h>
+#include <acpi/processor.h>
+#endif
+
+#include "powernow-k7.h"
+
+struct psb_s {
+ u8 signature[10];
+ u8 tableversion;
+ u8 flags;
+ u16 settlingtime;
+ u8 reserved1;
+ u8 numpst;
+};
+
+struct pst_s {
+ u32 cpuid;
+ u8 fsbspeed;
+ u8 maxfid;
+ u8 startvid;
+ u8 numpstates;
+};
+
+#ifdef CONFIG_X86_POWERNOW_K7_ACPI
+union powernow_acpi_control_t {
+ struct {
+ unsigned long fid:5,
+ vid:5,
+ sgtc:20,
+ res1:2;
+ } bits;
+ unsigned long val;
+};
+#endif
+
+/* divide by 1000 to get VCore voltage in V. */
+static const int mobile_vid_table[32] = {
+ 2000, 1950, 1900, 1850, 1800, 1750, 1700, 1650,
+ 1600, 1550, 1500, 1450, 1400, 1350, 1300, 0,
+ 1275, 1250, 1225, 1200, 1175, 1150, 1125, 1100,
+ 1075, 1050, 1025, 1000, 975, 950, 925, 0,
+};
+
+/* divide by 10 to get FID. */
+static const int fid_codes[32] = {
+ 110, 115, 120, 125, 50, 55, 60, 65,
+ 70, 75, 80, 85, 90, 95, 100, 105,
+ 30, 190, 40, 200, 130, 135, 140, 210,
+ 150, 225, 160, 165, 170, 180, -1, -1,
+};
+
+/* This parameter is used in order to force ACPI instead of legacy method for
+ * configuration purpose.
+ */
+
+static int acpi_force;
+
+static struct cpufreq_frequency_table *powernow_table;
+
+static unsigned int can_scale_bus;
+static unsigned int can_scale_vid;
+static unsigned int minimum_speed = -1;
+static unsigned int maximum_speed;
+static unsigned int number_scales;
+static unsigned int fsb;
+static unsigned int latency;
+static char have_a0;
+
+static int check_fsb(unsigned int fsbspeed)
+{
+ int delta;
+ unsigned int f = fsb / 1000;
+
+ delta = (fsbspeed > f) ? fsbspeed - f : f - fsbspeed;
+ return delta < 5;
+}
+
+static const struct x86_cpu_id powernow_k7_cpuids[] = {
+ X86_MATCH_VENDOR_FAM(AMD, 6, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, powernow_k7_cpuids);
+
+static int check_powernow(void)
+{
+ struct cpuinfo_x86 *c = &cpu_data(0);
+ unsigned int maxei, eax, ebx, ecx, edx;
+
+ if (!x86_match_cpu(powernow_k7_cpuids))
+ return 0;
+
+ /* Get maximum capabilities */
+ maxei = cpuid_eax(0x80000000);
+ if (maxei < 0x80000007) { /* Any powernow info ? */
+#ifdef MODULE
+ pr_info("No powernow capabilities detected\n");
+#endif
+ return 0;
+ }
+
+ if ((c->x86_model == 6) && (c->x86_stepping == 0)) {
+ pr_info("K7 660[A0] core detected, enabling errata workarounds\n");
+ have_a0 = 1;
+ }
+
+ cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
+
+ /* Check we can actually do something before we say anything.*/
+ if (!(edx & (1 << 1 | 1 << 2)))
+ return 0;
+
+ pr_info("PowerNOW! Technology present. Can scale: ");
+
+ if (edx & 1 << 1) {
+ pr_cont("frequency");
+ can_scale_bus = 1;
+ }
+
+ if ((edx & (1 << 1 | 1 << 2)) == 0x6)
+ pr_cont(" and ");
+
+ if (edx & 1 << 2) {
+ pr_cont("voltage");
+ can_scale_vid = 1;
+ }
+
+ pr_cont("\n");
+ return 1;
+}
+
+#ifdef CONFIG_X86_POWERNOW_K7_ACPI
+static void invalidate_entry(unsigned int entry)
+{
+ powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID;
+}
+#endif
+
+static int get_ranges(unsigned char *pst)
+{
+ unsigned int j;
+ unsigned int speed;
+ u8 fid, vid;
+
+ powernow_table = kzalloc((sizeof(*powernow_table) *
+ (number_scales + 1)), GFP_KERNEL);
+ if (!powernow_table)
+ return -ENOMEM;
+
+ for (j = 0 ; j < number_scales; j++) {
+ fid = *pst++;
+
+ powernow_table[j].frequency = (fsb * fid_codes[fid]) / 10;
+ powernow_table[j].driver_data = fid; /* lower 8 bits */
+
+ speed = powernow_table[j].frequency;
+
+ if ((fid_codes[fid] % 10) == 5) {
+#ifdef CONFIG_X86_POWERNOW_K7_ACPI
+ if (have_a0 == 1)
+ invalidate_entry(j);
+#endif
+ }
+
+ if (speed < minimum_speed)
+ minimum_speed = speed;
+ if (speed > maximum_speed)
+ maximum_speed = speed;
+
+ vid = *pst++;
+ powernow_table[j].driver_data |= (vid << 8); /* upper 8 bits */
+
+ pr_debug(" FID: 0x%x (%d.%dx [%dMHz]) "
+ "VID: 0x%x (%d.%03dV)\n", fid, fid_codes[fid] / 10,
+ fid_codes[fid] % 10, speed/1000, vid,
+ mobile_vid_table[vid]/1000,
+ mobile_vid_table[vid]%1000);
+ }
+ powernow_table[number_scales].frequency = CPUFREQ_TABLE_END;
+ powernow_table[number_scales].driver_data = 0;
+
+ return 0;
+}
+
+
+static void change_FID(int fid)
+{
+ union msr_fidvidctl fidvidctl;
+
+ rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val);
+ if (fidvidctl.bits.FID != fid) {
+ fidvidctl.bits.SGTC = latency;
+ fidvidctl.bits.FID = fid;
+ fidvidctl.bits.VIDC = 0;
+ fidvidctl.bits.FIDC = 1;
+ wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val);
+ }
+}
+
+
+static void change_VID(int vid)
+{
+ union msr_fidvidctl fidvidctl;
+
+ rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val);
+ if (fidvidctl.bits.VID != vid) {
+ fidvidctl.bits.SGTC = latency;
+ fidvidctl.bits.VID = vid;
+ fidvidctl.bits.FIDC = 0;
+ fidvidctl.bits.VIDC = 1;
+ wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val);
+ }
+}
+
+
+static int powernow_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ u8 fid, vid;
+ struct cpufreq_freqs freqs;
+ union msr_fidvidstatus fidvidstatus;
+ int cfid;
+
+ /* fid are the lower 8 bits of the index we stored into
+ * the cpufreq frequency table in powernow_decode_bios,
+ * vid are the upper 8 bits.
+ */
+
+ fid = powernow_table[index].driver_data & 0xFF;
+ vid = (powernow_table[index].driver_data & 0xFF00) >> 8;
+
+ rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val);
+ cfid = fidvidstatus.bits.CFID;
+ freqs.old = fsb * fid_codes[cfid] / 10;
+
+ freqs.new = powernow_table[index].frequency;
+
+ /* Now do the magic poking into the MSRs. */
+
+ if (have_a0 == 1) /* A0 errata 5 */
+ local_irq_disable();
+
+ if (freqs.old > freqs.new) {
+ /* Going down, so change FID first */
+ change_FID(fid);
+ change_VID(vid);
+ } else {
+ /* Going up, so change VID first */
+ change_VID(vid);
+ change_FID(fid);
+ }
+
+
+ if (have_a0 == 1)
+ local_irq_enable();
+
+ return 0;
+}
+
+
+#ifdef CONFIG_X86_POWERNOW_K7_ACPI
+
+static struct acpi_processor_performance *acpi_processor_perf;
+
+static int powernow_acpi_init(void)
+{
+ int i;
+ int retval = 0;
+ union powernow_acpi_control_t pc;
+
+ if (acpi_processor_perf != NULL && powernow_table != NULL) {
+ retval = -EINVAL;
+ goto err0;
+ }
+
+ acpi_processor_perf = kzalloc(sizeof(*acpi_processor_perf), GFP_KERNEL);
+ if (!acpi_processor_perf) {
+ retval = -ENOMEM;
+ goto err0;
+ }
+
+ if (!zalloc_cpumask_var(&acpi_processor_perf->shared_cpu_map,
+ GFP_KERNEL)) {
+ retval = -ENOMEM;
+ goto err05;
+ }
+
+ if (acpi_processor_register_performance(acpi_processor_perf, 0)) {
+ retval = -EIO;
+ goto err1;
+ }
+
+ if (acpi_processor_perf->control_register.space_id !=
+ ACPI_ADR_SPACE_FIXED_HARDWARE) {
+ retval = -ENODEV;
+ goto err2;
+ }
+
+ if (acpi_processor_perf->status_register.space_id !=
+ ACPI_ADR_SPACE_FIXED_HARDWARE) {
+ retval = -ENODEV;
+ goto err2;
+ }
+
+ number_scales = acpi_processor_perf->state_count;
+
+ if (number_scales < 2) {
+ retval = -ENODEV;
+ goto err2;
+ }
+
+ powernow_table = kzalloc((sizeof(*powernow_table) *
+ (number_scales + 1)), GFP_KERNEL);
+ if (!powernow_table) {
+ retval = -ENOMEM;
+ goto err2;
+ }
+
+ pc.val = (unsigned long) acpi_processor_perf->states[0].control;
+ for (i = 0; i < number_scales; i++) {
+ u8 fid, vid;
+ struct acpi_processor_px *state =
+ &acpi_processor_perf->states[i];
+ unsigned int speed, speed_mhz;
+
+ pc.val = (unsigned long) state->control;
+ pr_debug("acpi: P%d: %d MHz %d mW %d uS control %08x SGTC %d\n",
+ i,
+ (u32) state->core_frequency,
+ (u32) state->power,
+ (u32) state->transition_latency,
+ (u32) state->control,
+ pc.bits.sgtc);
+
+ vid = pc.bits.vid;
+ fid = pc.bits.fid;
+
+ powernow_table[i].frequency = fsb * fid_codes[fid] / 10;
+ powernow_table[i].driver_data = fid; /* lower 8 bits */
+ powernow_table[i].driver_data |= (vid << 8); /* upper 8 bits */
+
+ speed = powernow_table[i].frequency;
+ speed_mhz = speed / 1000;
+
+ /* processor_perflib will multiply the MHz value by 1000 to
+ * get a KHz value (e.g. 1266000). However, powernow-k7 works
+ * with true KHz values (e.g. 1266768). To ensure that all
+ * powernow frequencies are available, we must ensure that
+ * ACPI doesn't restrict them, so we round up the MHz value
+ * to ensure that perflib's computed KHz value is greater than
+ * or equal to powernow's KHz value.
+ */
+ if (speed % 1000 > 0)
+ speed_mhz++;
+
+ if ((fid_codes[fid] % 10) == 5) {
+ if (have_a0 == 1)
+ invalidate_entry(i);
+ }
+
+ pr_debug(" FID: 0x%x (%d.%dx [%dMHz]) "
+ "VID: 0x%x (%d.%03dV)\n", fid, fid_codes[fid] / 10,
+ fid_codes[fid] % 10, speed_mhz, vid,
+ mobile_vid_table[vid]/1000,
+ mobile_vid_table[vid]%1000);
+
+ if (state->core_frequency != speed_mhz) {
+ state->core_frequency = speed_mhz;
+ pr_debug(" Corrected ACPI frequency to %d\n",
+ speed_mhz);
+ }
+
+ if (latency < pc.bits.sgtc)
+ latency = pc.bits.sgtc;
+
+ if (speed < minimum_speed)
+ minimum_speed = speed;
+ if (speed > maximum_speed)
+ maximum_speed = speed;
+ }
+
+ powernow_table[i].frequency = CPUFREQ_TABLE_END;
+ powernow_table[i].driver_data = 0;
+
+ /* notify BIOS that we exist */
+ acpi_processor_notify_smm(THIS_MODULE);
+
+ return 0;
+
+err2:
+ acpi_processor_unregister_performance(0);
+err1:
+ free_cpumask_var(acpi_processor_perf->shared_cpu_map);
+err05:
+ kfree(acpi_processor_perf);
+err0:
+ pr_warn("ACPI perflib can not be used on this platform\n");
+ acpi_processor_perf = NULL;
+ return retval;
+}
+#else
+static int powernow_acpi_init(void)
+{
+ pr_info("no support for ACPI processor found - please recompile your kernel with ACPI processor\n");
+ return -EINVAL;
+}
+#endif
+
+static void print_pst_entry(struct pst_s *pst, unsigned int j)
+{
+ pr_debug("PST:%d (@%p)\n", j, pst);
+ pr_debug(" cpuid: 0x%x fsb: %d maxFID: 0x%x startvid: 0x%x\n",
+ pst->cpuid, pst->fsbspeed, pst->maxfid, pst->startvid);
+}
+
+static int powernow_decode_bios(int maxfid, int startvid)
+{
+ struct psb_s *psb;
+ struct pst_s *pst;
+ unsigned int i, j;
+ unsigned char *p;
+ unsigned int etuple;
+ unsigned int ret;
+
+ etuple = cpuid_eax(0x80000001);
+
+ for (i = 0xC0000; i < 0xffff0 ; i += 16) {
+
+ p = phys_to_virt(i);
+
+ if (memcmp(p, "AMDK7PNOW!", 10) == 0) {
+ pr_debug("Found PSB header at %p\n", p);
+ psb = (struct psb_s *) p;
+ pr_debug("Table version: 0x%x\n", psb->tableversion);
+ if (psb->tableversion != 0x12) {
+ pr_info("Sorry, only v1.2 tables supported right now\n");
+ return -ENODEV;
+ }
+
+ pr_debug("Flags: 0x%x\n", psb->flags);
+ if ((psb->flags & 1) == 0)
+ pr_debug("Mobile voltage regulator\n");
+ else
+ pr_debug("Desktop voltage regulator\n");
+
+ latency = psb->settlingtime;
+ if (latency < 100) {
+ pr_info("BIOS set settling time to %d microseconds. Should be at least 100. Correcting.\n",
+ latency);
+ latency = 100;
+ }
+ pr_debug("Settling Time: %d microseconds.\n",
+ psb->settlingtime);
+ pr_debug("Has %d PST tables. (Only dumping ones "
+ "relevant to this CPU).\n",
+ psb->numpst);
+
+ p += sizeof(*psb);
+
+ pst = (struct pst_s *) p;
+
+ for (j = 0; j < psb->numpst; j++) {
+ pst = (struct pst_s *) p;
+ number_scales = pst->numpstates;
+
+ if ((etuple == pst->cpuid) &&
+ check_fsb(pst->fsbspeed) &&
+ (maxfid == pst->maxfid) &&
+ (startvid == pst->startvid)) {
+ print_pst_entry(pst, j);
+ p = (char *)pst + sizeof(*pst);
+ ret = get_ranges(p);
+ return ret;
+ } else {
+ unsigned int k;
+ p = (char *)pst + sizeof(*pst);
+ for (k = 0; k < number_scales; k++)
+ p += 2;
+ }
+ }
+ pr_info("No PST tables match this cpuid (0x%x)\n",
+ etuple);
+ pr_info("This is indicative of a broken BIOS\n");
+
+ return -EINVAL;
+ }
+ p++;
+ }
+
+ return -ENODEV;
+}
+
+
+/*
+ * We use the fact that the bus frequency is somehow
+ * a multiple of 100000/3 khz, then we compute sgtc according
+ * to this multiple.
+ * That way, we match more how AMD thinks all of that work.
+ * We will then get the same kind of behaviour already tested under
+ * the "well-known" other OS.
+ */
+static int fixup_sgtc(void)
+{
+ unsigned int sgtc;
+ unsigned int m;
+
+ m = fsb / 3333;
+ if ((m % 10) >= 5)
+ m += 5;
+
+ m /= 10;
+
+ sgtc = 100 * m * latency;
+ sgtc = sgtc / 3;
+ if (sgtc > 0xfffff) {
+ pr_warn("SGTC too large %d\n", sgtc);
+ sgtc = 0xfffff;
+ }
+ return sgtc;
+}
+
+static unsigned int powernow_get(unsigned int cpu)
+{
+ union msr_fidvidstatus fidvidstatus;
+ unsigned int cfid;
+
+ if (cpu)
+ return 0;
+ rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val);
+ cfid = fidvidstatus.bits.CFID;
+
+ return fsb * fid_codes[cfid] / 10;
+}
+
+
+static int acer_cpufreq_pst(const struct dmi_system_id *d)
+{
+ pr_warn("%s laptop with broken PST tables in BIOS detected\n",
+ d->ident);
+ pr_warn("You need to downgrade to 3A21 (09/09/2002), or try a newer BIOS than 3A71 (01/20/2003)\n");
+ pr_warn("cpufreq scaling has been disabled as a result of this\n");
+ return 0;
+}
+
+/*
+ * Some Athlon laptops have really fucked PST tables.
+ * A BIOS update is all that can save them.
+ * Mention this, and disable cpufreq.
+ */
+static const struct dmi_system_id powernow_dmi_table[] = {
+ {
+ .callback = acer_cpufreq_pst,
+ .ident = "Acer Aspire",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "Insyde Software"),
+ DMI_MATCH(DMI_BIOS_VERSION, "3A71"),
+ },
+ },
+ { }
+};
+
+static int powernow_cpu_init(struct cpufreq_policy *policy)
+{
+ union msr_fidvidstatus fidvidstatus;
+ int result;
+
+ if (policy->cpu != 0)
+ return -ENODEV;
+
+ rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val);
+
+ recalibrate_cpu_khz();
+
+ fsb = (10 * cpu_khz) / fid_codes[fidvidstatus.bits.CFID];
+ if (!fsb) {
+ pr_warn("can not determine bus frequency\n");
+ return -EINVAL;
+ }
+ pr_debug("FSB: %3dMHz\n", fsb/1000);
+
+ if (dmi_check_system(powernow_dmi_table) || acpi_force) {
+ pr_info("PSB/PST known to be broken - trying ACPI instead\n");
+ result = powernow_acpi_init();
+ } else {
+ result = powernow_decode_bios(fidvidstatus.bits.MFID,
+ fidvidstatus.bits.SVID);
+ if (result) {
+ pr_info("Trying ACPI perflib\n");
+ maximum_speed = 0;
+ minimum_speed = -1;
+ latency = 0;
+ result = powernow_acpi_init();
+ if (result) {
+ pr_info("ACPI and legacy methods failed\n");
+ }
+ } else {
+ /* SGTC use the bus clock as timer */
+ latency = fixup_sgtc();
+ pr_info("SGTC: %d\n", latency);
+ }
+ }
+
+ if (result)
+ return result;
+
+ pr_info("Minimum speed %d MHz - Maximum speed %d MHz\n",
+ minimum_speed/1000, maximum_speed/1000);
+
+ policy->cpuinfo.transition_latency =
+ cpufreq_scale(2000000UL, fsb, latency);
+ policy->freq_table = powernow_table;
+
+ return 0;
+}
+
+static int powernow_cpu_exit(struct cpufreq_policy *policy)
+{
+#ifdef CONFIG_X86_POWERNOW_K7_ACPI
+ if (acpi_processor_perf) {
+ acpi_processor_unregister_performance(0);
+ free_cpumask_var(acpi_processor_perf->shared_cpu_map);
+ kfree(acpi_processor_perf);
+ }
+#endif
+
+ kfree(powernow_table);
+ return 0;
+}
+
+static struct cpufreq_driver powernow_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = powernow_target,
+ .get = powernow_get,
+#ifdef CONFIG_X86_POWERNOW_K7_ACPI
+ .bios_limit = acpi_processor_get_bios_limit,
+#endif
+ .init = powernow_cpu_init,
+ .exit = powernow_cpu_exit,
+ .name = "powernow-k7",
+ .attr = cpufreq_generic_attr,
+};
+
+static int __init powernow_init(void)
+{
+ if (check_powernow() == 0)
+ return -ENODEV;
+ return cpufreq_register_driver(&powernow_driver);
+}
+
+
+static void __exit powernow_exit(void)
+{
+ cpufreq_unregister_driver(&powernow_driver);
+}
+
+module_param(acpi_force, int, 0444);
+MODULE_PARM_DESC(acpi_force, "Force ACPI to be used.");
+
+MODULE_AUTHOR("Dave Jones");
+MODULE_DESCRIPTION("Powernow driver for AMD K7 processors.");
+MODULE_LICENSE("GPL");
+
+late_initcall(powernow_init);
+module_exit(powernow_exit);
+
diff --git a/drivers/cpufreq/powernow-k7.h b/drivers/cpufreq/powernow-k7.h
new file mode 100644
index 0000000000..4bc673faee
--- /dev/null
+++ b/drivers/cpufreq/powernow-k7.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * (C) 2003 Dave Jones.
+ *
+ * AMD-specific information
+ */
+
+union msr_fidvidctl {
+ struct {
+ unsigned FID:5, // 4:0
+ reserved1:3, // 7:5
+ VID:5, // 12:8
+ reserved2:3, // 15:13
+ FIDC:1, // 16
+ VIDC:1, // 17
+ reserved3:2, // 19:18
+ FIDCHGRATIO:1, // 20
+ reserved4:11, // 31-21
+ SGTC:20, // 32:51
+ reserved5:12; // 63:52
+ } bits;
+ unsigned long long val;
+};
+
+union msr_fidvidstatus {
+ struct {
+ unsigned CFID:5, // 4:0
+ reserved1:3, // 7:5
+ SFID:5, // 12:8
+ reserved2:3, // 15:13
+ MFID:5, // 20:16
+ reserved3:11, // 31:21
+ CVID:5, // 36:32
+ reserved4:3, // 39:37
+ SVID:5, // 44:40
+ reserved5:3, // 47:45
+ MVID:5, // 52:48
+ reserved6:11; // 63:53
+ } bits;
+ unsigned long long val;
+};
diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c
new file mode 100644
index 0000000000..b10f7a1b77
--- /dev/null
+++ b/drivers/cpufreq/powernow-k8.c
@@ -0,0 +1,1221 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * (c) 2003-2012 Advanced Micro Devices, Inc.
+ *
+ * Maintainer:
+ * Andreas Herrmann <herrmann.der.user@googlemail.com>
+ *
+ * Based on the powernow-k7.c module written by Dave Jones.
+ * (C) 2003 Dave Jones on behalf of SuSE Labs
+ * (C) 2004 Dominik Brodowski <linux@brodo.de>
+ * (C) 2004 Pavel Machek <pavel@ucw.cz>
+ * Based upon datasheets & sample CPUs kindly provided by AMD.
+ *
+ * Valuable input gratefully received from Dave Jones, Pavel Machek,
+ * Dominik Brodowski, Jacob Shin, and others.
+ * Originally developed by Paul Devriendt.
+ *
+ * Processor information obtained from Chapter 9 (Power and Thermal
+ * Management) of the "BIOS and Kernel Developer's Guide (BKDG) for
+ * the AMD Athlon 64 and AMD Opteron Processors" and section "2.x
+ * Power Management" in BKDGs for newer AMD CPU families.
+ *
+ * Tables for specific CPUs can be inferred from AMD's processor
+ * power and thermal data sheets, (e.g. 30417.pdf, 30430.pdf, 43375.pdf)
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/cpumask.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+
+#include <asm/msr.h>
+#include <asm/cpu_device_id.h>
+
+#include <linux/acpi.h>
+#include <linux/mutex.h>
+#include <acpi/processor.h>
+
+#define VERSION "version 2.20.00"
+#include "powernow-k8.h"
+
+/* serialize freq changes */
+static DEFINE_MUTEX(fidvid_mutex);
+
+static DEFINE_PER_CPU(struct powernow_k8_data *, powernow_data);
+
+static struct cpufreq_driver cpufreq_amd64_driver;
+
+/* Return a frequency in MHz, given an input fid */
+static u32 find_freq_from_fid(u32 fid)
+{
+ return 800 + (fid * 100);
+}
+
+/* Return a frequency in KHz, given an input fid */
+static u32 find_khz_freq_from_fid(u32 fid)
+{
+ return 1000 * find_freq_from_fid(fid);
+}
+
+/* Return the vco fid for an input fid
+ *
+ * Each "low" fid has corresponding "high" fid, and you can get to "low" fids
+ * only from corresponding high fids. This returns "high" fid corresponding to
+ * "low" one.
+ */
+static u32 convert_fid_to_vco_fid(u32 fid)
+{
+ if (fid < HI_FID_TABLE_BOTTOM)
+ return 8 + (2 * fid);
+ else
+ return fid;
+}
+
+/*
+ * Return 1 if the pending bit is set. Unless we just instructed the processor
+ * to transition to a new state, seeing this bit set is really bad news.
+ */
+static int pending_bit_stuck(void)
+{
+ u32 lo, hi __always_unused;
+
+ rdmsr(MSR_FIDVID_STATUS, lo, hi);
+ return lo & MSR_S_LO_CHANGE_PENDING ? 1 : 0;
+}
+
+/*
+ * Update the global current fid / vid values from the status msr.
+ * Returns 1 on error.
+ */
+static int query_current_values_with_pending_wait(struct powernow_k8_data *data)
+{
+ u32 lo, hi;
+ u32 i = 0;
+
+ do {
+ if (i++ > 10000) {
+ pr_debug("detected change pending stuck\n");
+ return 1;
+ }
+ rdmsr(MSR_FIDVID_STATUS, lo, hi);
+ } while (lo & MSR_S_LO_CHANGE_PENDING);
+
+ data->currvid = hi & MSR_S_HI_CURRENT_VID;
+ data->currfid = lo & MSR_S_LO_CURRENT_FID;
+
+ return 0;
+}
+
+/* the isochronous relief time */
+static void count_off_irt(struct powernow_k8_data *data)
+{
+ udelay((1 << data->irt) * 10);
+}
+
+/* the voltage stabilization time */
+static void count_off_vst(struct powernow_k8_data *data)
+{
+ udelay(data->vstable * VST_UNITS_20US);
+}
+
+/* need to init the control msr to a safe value (for each cpu) */
+static void fidvid_msr_init(void)
+{
+ u32 lo, hi;
+ u8 fid, vid;
+
+ rdmsr(MSR_FIDVID_STATUS, lo, hi);
+ vid = hi & MSR_S_HI_CURRENT_VID;
+ fid = lo & MSR_S_LO_CURRENT_FID;
+ lo = fid | (vid << MSR_C_LO_VID_SHIFT);
+ hi = MSR_C_HI_STP_GNT_BENIGN;
+ pr_debug("cpu%d, init lo 0x%x, hi 0x%x\n", smp_processor_id(), lo, hi);
+ wrmsr(MSR_FIDVID_CTL, lo, hi);
+}
+
+/* write the new fid value along with the other control fields to the msr */
+static int write_new_fid(struct powernow_k8_data *data, u32 fid)
+{
+ u32 lo;
+ u32 savevid = data->currvid;
+ u32 i = 0;
+
+ if ((fid & INVALID_FID_MASK) || (data->currvid & INVALID_VID_MASK)) {
+ pr_err("internal error - overflow on fid write\n");
+ return 1;
+ }
+
+ lo = fid;
+ lo |= (data->currvid << MSR_C_LO_VID_SHIFT);
+ lo |= MSR_C_LO_INIT_FID_VID;
+
+ pr_debug("writing fid 0x%x, lo 0x%x, hi 0x%x\n",
+ fid, lo, data->plllock * PLL_LOCK_CONVERSION);
+
+ do {
+ wrmsr(MSR_FIDVID_CTL, lo, data->plllock * PLL_LOCK_CONVERSION);
+ if (i++ > 100) {
+ pr_err("Hardware error - pending bit very stuck - no further pstate changes possible\n");
+ return 1;
+ }
+ } while (query_current_values_with_pending_wait(data));
+
+ count_off_irt(data);
+
+ if (savevid != data->currvid) {
+ pr_err("vid change on fid trans, old 0x%x, new 0x%x\n",
+ savevid, data->currvid);
+ return 1;
+ }
+
+ if (fid != data->currfid) {
+ pr_err("fid trans failed, fid 0x%x, curr 0x%x\n", fid,
+ data->currfid);
+ return 1;
+ }
+
+ return 0;
+}
+
+/* Write a new vid to the hardware */
+static int write_new_vid(struct powernow_k8_data *data, u32 vid)
+{
+ u32 lo;
+ u32 savefid = data->currfid;
+ int i = 0;
+
+ if ((data->currfid & INVALID_FID_MASK) || (vid & INVALID_VID_MASK)) {
+ pr_err("internal error - overflow on vid write\n");
+ return 1;
+ }
+
+ lo = data->currfid;
+ lo |= (vid << MSR_C_LO_VID_SHIFT);
+ lo |= MSR_C_LO_INIT_FID_VID;
+
+ pr_debug("writing vid 0x%x, lo 0x%x, hi 0x%x\n",
+ vid, lo, STOP_GRANT_5NS);
+
+ do {
+ wrmsr(MSR_FIDVID_CTL, lo, STOP_GRANT_5NS);
+ if (i++ > 100) {
+ pr_err("internal error - pending bit very stuck - no further pstate changes possible\n");
+ return 1;
+ }
+ } while (query_current_values_with_pending_wait(data));
+
+ if (savefid != data->currfid) {
+ pr_err("fid changed on vid trans, old 0x%x new 0x%x\n",
+ savefid, data->currfid);
+ return 1;
+ }
+
+ if (vid != data->currvid) {
+ pr_err("vid trans failed, vid 0x%x, curr 0x%x\n",
+ vid, data->currvid);
+ return 1;
+ }
+
+ return 0;
+}
+
+/*
+ * Reduce the vid by the max of step or reqvid.
+ * Decreasing vid codes represent increasing voltages:
+ * vid of 0 is 1.550V, vid of 0x1e is 0.800V, vid of VID_OFF is off.
+ */
+static int decrease_vid_code_by_step(struct powernow_k8_data *data,
+ u32 reqvid, u32 step)
+{
+ if ((data->currvid - reqvid) > step)
+ reqvid = data->currvid - step;
+
+ if (write_new_vid(data, reqvid))
+ return 1;
+
+ count_off_vst(data);
+
+ return 0;
+}
+
+/* Change Opteron/Athlon64 fid and vid, by the 3 phases. */
+static int transition_fid_vid(struct powernow_k8_data *data,
+ u32 reqfid, u32 reqvid)
+{
+ if (core_voltage_pre_transition(data, reqvid, reqfid))
+ return 1;
+
+ if (core_frequency_transition(data, reqfid))
+ return 1;
+
+ if (core_voltage_post_transition(data, reqvid))
+ return 1;
+
+ if (query_current_values_with_pending_wait(data))
+ return 1;
+
+ if ((reqfid != data->currfid) || (reqvid != data->currvid)) {
+ pr_err("failed (cpu%d): req 0x%x 0x%x, curr 0x%x 0x%x\n",
+ smp_processor_id(),
+ reqfid, reqvid, data->currfid, data->currvid);
+ return 1;
+ }
+
+ pr_debug("transitioned (cpu%d): new fid 0x%x, vid 0x%x\n",
+ smp_processor_id(), data->currfid, data->currvid);
+
+ return 0;
+}
+
+/* Phase 1 - core voltage transition ... setup voltage */
+static int core_voltage_pre_transition(struct powernow_k8_data *data,
+ u32 reqvid, u32 reqfid)
+{
+ u32 rvosteps = data->rvo;
+ u32 savefid = data->currfid;
+ u32 maxvid, lo __always_unused, rvomult = 1;
+
+ pr_debug("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, reqvid 0x%x, rvo 0x%x\n",
+ smp_processor_id(),
+ data->currfid, data->currvid, reqvid, data->rvo);
+
+ if ((savefid < LO_FID_TABLE_TOP) && (reqfid < LO_FID_TABLE_TOP))
+ rvomult = 2;
+ rvosteps *= rvomult;
+ rdmsr(MSR_FIDVID_STATUS, lo, maxvid);
+ maxvid = 0x1f & (maxvid >> 16);
+ pr_debug("ph1 maxvid=0x%x\n", maxvid);
+ if (reqvid < maxvid) /* lower numbers are higher voltages */
+ reqvid = maxvid;
+
+ while (data->currvid > reqvid) {
+ pr_debug("ph1: curr 0x%x, req vid 0x%x\n",
+ data->currvid, reqvid);
+ if (decrease_vid_code_by_step(data, reqvid, data->vidmvs))
+ return 1;
+ }
+
+ while ((rvosteps > 0) &&
+ ((rvomult * data->rvo + data->currvid) > reqvid)) {
+ if (data->currvid == maxvid) {
+ rvosteps = 0;
+ } else {
+ pr_debug("ph1: changing vid for rvo, req 0x%x\n",
+ data->currvid - 1);
+ if (decrease_vid_code_by_step(data, data->currvid-1, 1))
+ return 1;
+ rvosteps--;
+ }
+ }
+
+ if (query_current_values_with_pending_wait(data))
+ return 1;
+
+ if (savefid != data->currfid) {
+ pr_err("ph1 err, currfid changed 0x%x\n", data->currfid);
+ return 1;
+ }
+
+ pr_debug("ph1 complete, currfid 0x%x, currvid 0x%x\n",
+ data->currfid, data->currvid);
+
+ return 0;
+}
+
+/* Phase 2 - core frequency transition */
+static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid)
+{
+ u32 vcoreqfid, vcocurrfid, vcofiddiff;
+ u32 fid_interval, savevid = data->currvid;
+
+ if (data->currfid == reqfid) {
+ pr_err("ph2 null fid transition 0x%x\n", data->currfid);
+ return 0;
+ }
+
+ pr_debug("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, reqfid 0x%x\n",
+ smp_processor_id(),
+ data->currfid, data->currvid, reqfid);
+
+ vcoreqfid = convert_fid_to_vco_fid(reqfid);
+ vcocurrfid = convert_fid_to_vco_fid(data->currfid);
+ vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid
+ : vcoreqfid - vcocurrfid;
+
+ if ((reqfid <= LO_FID_TABLE_TOP) && (data->currfid <= LO_FID_TABLE_TOP))
+ vcofiddiff = 0;
+
+ while (vcofiddiff > 2) {
+ (data->currfid & 1) ? (fid_interval = 1) : (fid_interval = 2);
+
+ if (reqfid > data->currfid) {
+ if (data->currfid > LO_FID_TABLE_TOP) {
+ if (write_new_fid(data,
+ data->currfid + fid_interval))
+ return 1;
+ } else {
+ if (write_new_fid
+ (data,
+ 2 + convert_fid_to_vco_fid(data->currfid)))
+ return 1;
+ }
+ } else {
+ if (write_new_fid(data, data->currfid - fid_interval))
+ return 1;
+ }
+
+ vcocurrfid = convert_fid_to_vco_fid(data->currfid);
+ vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid
+ : vcoreqfid - vcocurrfid;
+ }
+
+ if (write_new_fid(data, reqfid))
+ return 1;
+
+ if (query_current_values_with_pending_wait(data))
+ return 1;
+
+ if (data->currfid != reqfid) {
+ pr_err("ph2: mismatch, failed fid transition, curr 0x%x, req 0x%x\n",
+ data->currfid, reqfid);
+ return 1;
+ }
+
+ if (savevid != data->currvid) {
+ pr_err("ph2: vid changed, save 0x%x, curr 0x%x\n",
+ savevid, data->currvid);
+ return 1;
+ }
+
+ pr_debug("ph2 complete, currfid 0x%x, currvid 0x%x\n",
+ data->currfid, data->currvid);
+
+ return 0;
+}
+
+/* Phase 3 - core voltage transition flow ... jump to the final vid. */
+static int core_voltage_post_transition(struct powernow_k8_data *data,
+ u32 reqvid)
+{
+ u32 savefid = data->currfid;
+ u32 savereqvid = reqvid;
+
+ pr_debug("ph3 (cpu%d): starting, currfid 0x%x, currvid 0x%x\n",
+ smp_processor_id(),
+ data->currfid, data->currvid);
+
+ if (reqvid != data->currvid) {
+ if (write_new_vid(data, reqvid))
+ return 1;
+
+ if (savefid != data->currfid) {
+ pr_err("ph3: bad fid change, save 0x%x, curr 0x%x\n",
+ savefid, data->currfid);
+ return 1;
+ }
+
+ if (data->currvid != reqvid) {
+ pr_err("ph3: failed vid transition\n, req 0x%x, curr 0x%x",
+ reqvid, data->currvid);
+ return 1;
+ }
+ }
+
+ if (query_current_values_with_pending_wait(data))
+ return 1;
+
+ if (savereqvid != data->currvid) {
+ pr_debug("ph3 failed, currvid 0x%x\n", data->currvid);
+ return 1;
+ }
+
+ if (savefid != data->currfid) {
+ pr_debug("ph3 failed, currfid changed 0x%x\n",
+ data->currfid);
+ return 1;
+ }
+
+ pr_debug("ph3 complete, currfid 0x%x, currvid 0x%x\n",
+ data->currfid, data->currvid);
+
+ return 0;
+}
+
+static const struct x86_cpu_id powernow_k8_ids[] = {
+ /* IO based frequency switching */
+ X86_MATCH_VENDOR_FAM(AMD, 0xf, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, powernow_k8_ids);
+
+static void check_supported_cpu(void *_rc)
+{
+ u32 eax, ebx, ecx, edx;
+ int *rc = _rc;
+
+ *rc = -ENODEV;
+
+ eax = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
+
+ if ((eax & CPUID_XFAM) == CPUID_XFAM_K8) {
+ if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) ||
+ ((eax & CPUID_XMOD) > CPUID_XMOD_REV_MASK)) {
+ pr_info("Processor cpuid %x not supported\n", eax);
+ return;
+ }
+
+ eax = cpuid_eax(CPUID_GET_MAX_CAPABILITIES);
+ if (eax < CPUID_FREQ_VOLT_CAPABILITIES) {
+ pr_info("No frequency change capabilities detected\n");
+ return;
+ }
+
+ cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx);
+ if ((edx & P_STATE_TRANSITION_CAPABLE)
+ != P_STATE_TRANSITION_CAPABLE) {
+ pr_info("Power state transitions not supported\n");
+ return;
+ }
+ *rc = 0;
+ }
+}
+
+static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst,
+ u8 maxvid)
+{
+ unsigned int j;
+ u8 lastfid = 0xff;
+
+ for (j = 0; j < data->numps; j++) {
+ if (pst[j].vid > LEAST_VID) {
+ pr_err(FW_BUG "vid %d invalid : 0x%x\n", j,
+ pst[j].vid);
+ return -EINVAL;
+ }
+ if (pst[j].vid < data->rvo) {
+ /* vid + rvo >= 0 */
+ pr_err(FW_BUG "0 vid exceeded with pstate %d\n", j);
+ return -ENODEV;
+ }
+ if (pst[j].vid < maxvid + data->rvo) {
+ /* vid + rvo >= maxvid */
+ pr_err(FW_BUG "maxvid exceeded with pstate %d\n", j);
+ return -ENODEV;
+ }
+ if (pst[j].fid > MAX_FID) {
+ pr_err(FW_BUG "maxfid exceeded with pstate %d\n", j);
+ return -ENODEV;
+ }
+ if (j && (pst[j].fid < HI_FID_TABLE_BOTTOM)) {
+ /* Only first fid is allowed to be in "low" range */
+ pr_err(FW_BUG "two low fids - %d : 0x%x\n", j,
+ pst[j].fid);
+ return -EINVAL;
+ }
+ if (pst[j].fid < lastfid)
+ lastfid = pst[j].fid;
+ }
+ if (lastfid & 1) {
+ pr_err(FW_BUG "lastfid invalid\n");
+ return -EINVAL;
+ }
+ if (lastfid > LO_FID_TABLE_TOP)
+ pr_info(FW_BUG "first fid not from lo freq table\n");
+
+ return 0;
+}
+
+static void invalidate_entry(struct cpufreq_frequency_table *powernow_table,
+ unsigned int entry)
+{
+ powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID;
+}
+
+static void print_basics(struct powernow_k8_data *data)
+{
+ int j;
+ for (j = 0; j < data->numps; j++) {
+ if (data->powernow_table[j].frequency !=
+ CPUFREQ_ENTRY_INVALID) {
+ pr_info("fid 0x%x (%d MHz), vid 0x%x\n",
+ data->powernow_table[j].driver_data & 0xff,
+ data->powernow_table[j].frequency/1000,
+ data->powernow_table[j].driver_data >> 8);
+ }
+ }
+ if (data->batps)
+ pr_info("Only %d pstates on battery\n", data->batps);
+}
+
+static int fill_powernow_table(struct powernow_k8_data *data,
+ struct pst_s *pst, u8 maxvid)
+{
+ struct cpufreq_frequency_table *powernow_table;
+ unsigned int j;
+
+ if (data->batps) {
+ /* use ACPI support to get full speed on mains power */
+ pr_warn("Only %d pstates usable (use ACPI driver for full range\n",
+ data->batps);
+ data->numps = data->batps;
+ }
+
+ for (j = 1; j < data->numps; j++) {
+ if (pst[j-1].fid >= pst[j].fid) {
+ pr_err("PST out of sequence\n");
+ return -EINVAL;
+ }
+ }
+
+ if (data->numps < 2) {
+ pr_err("no p states to transition\n");
+ return -ENODEV;
+ }
+
+ if (check_pst_table(data, pst, maxvid))
+ return -EINVAL;
+
+ powernow_table = kzalloc((sizeof(*powernow_table)
+ * (data->numps + 1)), GFP_KERNEL);
+ if (!powernow_table)
+ return -ENOMEM;
+
+ for (j = 0; j < data->numps; j++) {
+ int freq;
+ powernow_table[j].driver_data = pst[j].fid; /* lower 8 bits */
+ powernow_table[j].driver_data |= (pst[j].vid << 8); /* upper 8 bits */
+ freq = find_khz_freq_from_fid(pst[j].fid);
+ powernow_table[j].frequency = freq;
+ }
+ powernow_table[data->numps].frequency = CPUFREQ_TABLE_END;
+ powernow_table[data->numps].driver_data = 0;
+
+ if (query_current_values_with_pending_wait(data)) {
+ kfree(powernow_table);
+ return -EIO;
+ }
+
+ pr_debug("cfid 0x%x, cvid 0x%x\n", data->currfid, data->currvid);
+ data->powernow_table = powernow_table;
+ if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu)
+ print_basics(data);
+
+ for (j = 0; j < data->numps; j++)
+ if ((pst[j].fid == data->currfid) &&
+ (pst[j].vid == data->currvid))
+ return 0;
+
+ pr_debug("currfid/vid do not match PST, ignoring\n");
+ return 0;
+}
+
+/* Find and validate the PSB/PST table in BIOS. */
+static int find_psb_table(struct powernow_k8_data *data)
+{
+ struct psb_s *psb;
+ unsigned int i;
+ u32 mvs;
+ u8 maxvid;
+ u32 cpst = 0;
+ u32 thiscpuid;
+
+ for (i = 0xc0000; i < 0xffff0; i += 0x10) {
+ /* Scan BIOS looking for the signature. */
+ /* It can not be at ffff0 - it is too big. */
+
+ psb = phys_to_virt(i);
+ if (memcmp(psb, PSB_ID_STRING, PSB_ID_STRING_LEN) != 0)
+ continue;
+
+ pr_debug("found PSB header at 0x%p\n", psb);
+
+ pr_debug("table vers: 0x%x\n", psb->tableversion);
+ if (psb->tableversion != PSB_VERSION_1_4) {
+ pr_err(FW_BUG "PSB table is not v1.4\n");
+ return -ENODEV;
+ }
+
+ pr_debug("flags: 0x%x\n", psb->flags1);
+ if (psb->flags1) {
+ pr_err(FW_BUG "unknown flags\n");
+ return -ENODEV;
+ }
+
+ data->vstable = psb->vstable;
+ pr_debug("voltage stabilization time: %d(*20us)\n",
+ data->vstable);
+
+ pr_debug("flags2: 0x%x\n", psb->flags2);
+ data->rvo = psb->flags2 & 3;
+ data->irt = ((psb->flags2) >> 2) & 3;
+ mvs = ((psb->flags2) >> 4) & 3;
+ data->vidmvs = 1 << mvs;
+ data->batps = ((psb->flags2) >> 6) & 3;
+
+ pr_debug("ramp voltage offset: %d\n", data->rvo);
+ pr_debug("isochronous relief time: %d\n", data->irt);
+ pr_debug("maximum voltage step: %d - 0x%x\n", mvs, data->vidmvs);
+
+ pr_debug("numpst: 0x%x\n", psb->num_tables);
+ cpst = psb->num_tables;
+ if ((psb->cpuid == 0x00000fc0) ||
+ (psb->cpuid == 0x00000fe0)) {
+ thiscpuid = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
+ if ((thiscpuid == 0x00000fc0) ||
+ (thiscpuid == 0x00000fe0))
+ cpst = 1;
+ }
+ if (cpst != 1) {
+ pr_err(FW_BUG "numpst must be 1\n");
+ return -ENODEV;
+ }
+
+ data->plllock = psb->plllocktime;
+ pr_debug("plllocktime: 0x%x (units 1us)\n", psb->plllocktime);
+ pr_debug("maxfid: 0x%x\n", psb->maxfid);
+ pr_debug("maxvid: 0x%x\n", psb->maxvid);
+ maxvid = psb->maxvid;
+
+ data->numps = psb->numps;
+ pr_debug("numpstates: 0x%x\n", data->numps);
+ return fill_powernow_table(data,
+ (struct pst_s *)(psb+1), maxvid);
+ }
+ /*
+ * If you see this message, complain to BIOS manufacturer. If
+ * he tells you "we do not support Linux" or some similar
+ * nonsense, remember that Windows 2000 uses the same legacy
+ * mechanism that the old Linux PSB driver uses. Tell them it
+ * is broken with Windows 2000.
+ *
+ * The reference to the AMD documentation is chapter 9 in the
+ * BIOS and Kernel Developer's Guide, which is available on
+ * www.amd.com
+ */
+ pr_err(FW_BUG "No PSB or ACPI _PSS objects\n");
+ pr_err("Make sure that your BIOS is up to date and Cool'N'Quiet support is enabled in BIOS setup\n");
+ return -ENODEV;
+}
+
+static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data,
+ unsigned int index)
+{
+ u64 control;
+
+ if (!data->acpi_data.state_count)
+ return;
+
+ control = data->acpi_data.states[index].control;
+ data->irt = (control >> IRT_SHIFT) & IRT_MASK;
+ data->rvo = (control >> RVO_SHIFT) & RVO_MASK;
+ data->exttype = (control >> EXT_TYPE_SHIFT) & EXT_TYPE_MASK;
+ data->plllock = (control >> PLL_L_SHIFT) & PLL_L_MASK;
+ data->vidmvs = 1 << ((control >> MVS_SHIFT) & MVS_MASK);
+ data->vstable = (control >> VST_SHIFT) & VST_MASK;
+}
+
+static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
+{
+ struct cpufreq_frequency_table *powernow_table;
+ int ret_val = -ENODEV;
+ u64 control, status;
+
+ if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) {
+ pr_debug("register performance failed: bad ACPI data\n");
+ return -EIO;
+ }
+
+ /* verify the data contained in the ACPI structures */
+ if (data->acpi_data.state_count <= 1) {
+ pr_debug("No ACPI P-States\n");
+ goto err_out;
+ }
+
+ control = data->acpi_data.control_register.space_id;
+ status = data->acpi_data.status_register.space_id;
+
+ if ((control != ACPI_ADR_SPACE_FIXED_HARDWARE) ||
+ (status != ACPI_ADR_SPACE_FIXED_HARDWARE)) {
+ pr_debug("Invalid control/status registers (%llx - %llx)\n",
+ control, status);
+ goto err_out;
+ }
+
+ /* fill in data->powernow_table */
+ powernow_table = kzalloc((sizeof(*powernow_table)
+ * (data->acpi_data.state_count + 1)), GFP_KERNEL);
+ if (!powernow_table)
+ goto err_out;
+
+ /* fill in data */
+ data->numps = data->acpi_data.state_count;
+ powernow_k8_acpi_pst_values(data, 0);
+
+ ret_val = fill_powernow_table_fidvid(data, powernow_table);
+ if (ret_val)
+ goto err_out_mem;
+
+ powernow_table[data->acpi_data.state_count].frequency =
+ CPUFREQ_TABLE_END;
+ data->powernow_table = powernow_table;
+
+ if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu)
+ print_basics(data);
+
+ /* notify BIOS that we exist */
+ acpi_processor_notify_smm(THIS_MODULE);
+
+ if (!zalloc_cpumask_var(&data->acpi_data.shared_cpu_map, GFP_KERNEL)) {
+ pr_err("unable to alloc powernow_k8_data cpumask\n");
+ ret_val = -ENOMEM;
+ goto err_out_mem;
+ }
+
+ return 0;
+
+err_out_mem:
+ kfree(powernow_table);
+
+err_out:
+ acpi_processor_unregister_performance(data->cpu);
+
+ /* data->acpi_data.state_count informs us at ->exit()
+ * whether ACPI was used */
+ data->acpi_data.state_count = 0;
+
+ return ret_val;
+}
+
+static int fill_powernow_table_fidvid(struct powernow_k8_data *data,
+ struct cpufreq_frequency_table *powernow_table)
+{
+ int i;
+
+ for (i = 0; i < data->acpi_data.state_count; i++) {
+ u32 fid;
+ u32 vid;
+ u32 freq, index;
+ u64 status, control;
+
+ if (data->exttype) {
+ status = data->acpi_data.states[i].status;
+ fid = status & EXT_FID_MASK;
+ vid = (status >> VID_SHIFT) & EXT_VID_MASK;
+ } else {
+ control = data->acpi_data.states[i].control;
+ fid = control & FID_MASK;
+ vid = (control >> VID_SHIFT) & VID_MASK;
+ }
+
+ pr_debug(" %d : fid 0x%x, vid 0x%x\n", i, fid, vid);
+
+ index = fid | (vid<<8);
+ powernow_table[i].driver_data = index;
+
+ freq = find_khz_freq_from_fid(fid);
+ powernow_table[i].frequency = freq;
+
+ /* verify frequency is OK */
+ if ((freq > (MAX_FREQ * 1000)) || (freq < (MIN_FREQ * 1000))) {
+ pr_debug("invalid freq %u kHz, ignoring\n", freq);
+ invalidate_entry(powernow_table, i);
+ continue;
+ }
+
+ /* verify voltage is OK -
+ * BIOSs are using "off" to indicate invalid */
+ if (vid == VID_OFF) {
+ pr_debug("invalid vid %u, ignoring\n", vid);
+ invalidate_entry(powernow_table, i);
+ continue;
+ }
+
+ if (freq != (data->acpi_data.states[i].core_frequency * 1000)) {
+ pr_info("invalid freq entries %u kHz vs. %u kHz\n",
+ freq, (unsigned int)
+ (data->acpi_data.states[i].core_frequency
+ * 1000));
+ invalidate_entry(powernow_table, i);
+ continue;
+ }
+ }
+ return 0;
+}
+
+static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data)
+{
+ if (data->acpi_data.state_count)
+ acpi_processor_unregister_performance(data->cpu);
+ free_cpumask_var(data->acpi_data.shared_cpu_map);
+}
+
+static int get_transition_latency(struct powernow_k8_data *data)
+{
+ int max_latency = 0;
+ int i;
+ for (i = 0; i < data->acpi_data.state_count; i++) {
+ int cur_latency = data->acpi_data.states[i].transition_latency
+ + data->acpi_data.states[i].bus_master_latency;
+ if (cur_latency > max_latency)
+ max_latency = cur_latency;
+ }
+ if (max_latency == 0) {
+ pr_err(FW_WARN "Invalid zero transition latency\n");
+ max_latency = 1;
+ }
+ /* value in usecs, needs to be in nanoseconds */
+ return 1000 * max_latency;
+}
+
+/* Take a frequency, and issue the fid/vid transition command */
+static int transition_frequency_fidvid(struct powernow_k8_data *data,
+ unsigned int index,
+ struct cpufreq_policy *policy)
+{
+ u32 fid = 0;
+ u32 vid = 0;
+ int res;
+ struct cpufreq_freqs freqs;
+
+ pr_debug("cpu %d transition to index %u\n", smp_processor_id(), index);
+
+ /* fid/vid correctness check for k8 */
+ /* fid are the lower 8 bits of the index we stored into
+ * the cpufreq frequency table in find_psb_table, vid
+ * are the upper 8 bits.
+ */
+ fid = data->powernow_table[index].driver_data & 0xFF;
+ vid = (data->powernow_table[index].driver_data & 0xFF00) >> 8;
+
+ pr_debug("table matched fid 0x%x, giving vid 0x%x\n", fid, vid);
+
+ if (query_current_values_with_pending_wait(data))
+ return 1;
+
+ if ((data->currvid == vid) && (data->currfid == fid)) {
+ pr_debug("target matches current values (fid 0x%x, vid 0x%x)\n",
+ fid, vid);
+ return 0;
+ }
+
+ pr_debug("cpu %d, changing to fid 0x%x, vid 0x%x\n",
+ smp_processor_id(), fid, vid);
+ freqs.old = find_khz_freq_from_fid(data->currfid);
+ freqs.new = find_khz_freq_from_fid(fid);
+
+ cpufreq_freq_transition_begin(policy, &freqs);
+ res = transition_fid_vid(data, fid, vid);
+ cpufreq_freq_transition_end(policy, &freqs, res);
+
+ return res;
+}
+
+struct powernowk8_target_arg {
+ struct cpufreq_policy *pol;
+ unsigned newstate;
+};
+
+static long powernowk8_target_fn(void *arg)
+{
+ struct powernowk8_target_arg *pta = arg;
+ struct cpufreq_policy *pol = pta->pol;
+ unsigned newstate = pta->newstate;
+ struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu);
+ u32 checkfid;
+ u32 checkvid;
+ int ret;
+
+ if (!data)
+ return -EINVAL;
+
+ checkfid = data->currfid;
+ checkvid = data->currvid;
+
+ if (pending_bit_stuck()) {
+ pr_err("failing targ, change pending bit set\n");
+ return -EIO;
+ }
+
+ pr_debug("targ: cpu %d, %d kHz, min %d, max %d\n",
+ pol->cpu, data->powernow_table[newstate].frequency, pol->min,
+ pol->max);
+
+ if (query_current_values_with_pending_wait(data))
+ return -EIO;
+
+ pr_debug("targ: curr fid 0x%x, vid 0x%x\n",
+ data->currfid, data->currvid);
+
+ if ((checkvid != data->currvid) ||
+ (checkfid != data->currfid)) {
+ pr_info("error - out of sync, fix 0x%x 0x%x, vid 0x%x 0x%x\n",
+ checkfid, data->currfid,
+ checkvid, data->currvid);
+ }
+
+ mutex_lock(&fidvid_mutex);
+
+ powernow_k8_acpi_pst_values(data, newstate);
+
+ ret = transition_frequency_fidvid(data, newstate, pol);
+
+ if (ret) {
+ pr_err("transition frequency failed\n");
+ mutex_unlock(&fidvid_mutex);
+ return 1;
+ }
+ mutex_unlock(&fidvid_mutex);
+
+ pol->cur = find_khz_freq_from_fid(data->currfid);
+
+ return 0;
+}
+
+/* Driver entry point to switch to the target frequency */
+static int powernowk8_target(struct cpufreq_policy *pol, unsigned index)
+{
+ struct powernowk8_target_arg pta = { .pol = pol, .newstate = index };
+
+ return work_on_cpu(pol->cpu, powernowk8_target_fn, &pta);
+}
+
+struct init_on_cpu {
+ struct powernow_k8_data *data;
+ int rc;
+};
+
+static void powernowk8_cpu_init_on_cpu(void *_init_on_cpu)
+{
+ struct init_on_cpu *init_on_cpu = _init_on_cpu;
+
+ if (pending_bit_stuck()) {
+ pr_err("failing init, change pending bit set\n");
+ init_on_cpu->rc = -ENODEV;
+ return;
+ }
+
+ if (query_current_values_with_pending_wait(init_on_cpu->data)) {
+ init_on_cpu->rc = -ENODEV;
+ return;
+ }
+
+ fidvid_msr_init();
+
+ init_on_cpu->rc = 0;
+}
+
+#define MISSING_PSS_MSG \
+ FW_BUG "No compatible ACPI _PSS objects found.\n" \
+ FW_BUG "First, make sure Cool'N'Quiet is enabled in the BIOS.\n" \
+ FW_BUG "If that doesn't help, try upgrading your BIOS.\n"
+
+/* per CPU init entry point to the driver */
+static int powernowk8_cpu_init(struct cpufreq_policy *pol)
+{
+ struct powernow_k8_data *data;
+ struct init_on_cpu init_on_cpu;
+ int rc, cpu;
+
+ smp_call_function_single(pol->cpu, check_supported_cpu, &rc, 1);
+ if (rc)
+ return -ENODEV;
+
+ data = kzalloc(sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ data->cpu = pol->cpu;
+
+ if (powernow_k8_cpu_init_acpi(data)) {
+ /*
+ * Use the PSB BIOS structure. This is only available on
+ * an UP version, and is deprecated by AMD.
+ */
+ if (num_online_cpus() != 1) {
+ pr_err_once(MISSING_PSS_MSG);
+ goto err_out;
+ }
+ if (pol->cpu != 0) {
+ pr_err(FW_BUG "No ACPI _PSS objects for CPU other than CPU0. Complain to your BIOS vendor.\n");
+ goto err_out;
+ }
+ rc = find_psb_table(data);
+ if (rc)
+ goto err_out;
+
+ /* Take a crude guess here.
+ * That guess was in microseconds, so multiply with 1000 */
+ pol->cpuinfo.transition_latency = (
+ ((data->rvo + 8) * data->vstable * VST_UNITS_20US) +
+ ((1 << data->irt) * 30)) * 1000;
+ } else /* ACPI _PSS objects available */
+ pol->cpuinfo.transition_latency = get_transition_latency(data);
+
+ /* only run on specific CPU from here on */
+ init_on_cpu.data = data;
+ smp_call_function_single(data->cpu, powernowk8_cpu_init_on_cpu,
+ &init_on_cpu, 1);
+ rc = init_on_cpu.rc;
+ if (rc != 0)
+ goto err_out_exit_acpi;
+
+ cpumask_copy(pol->cpus, topology_core_cpumask(pol->cpu));
+ data->available_cores = pol->cpus;
+ pol->freq_table = data->powernow_table;
+
+ pr_debug("cpu_init done, current fid 0x%x, vid 0x%x\n",
+ data->currfid, data->currvid);
+
+ /* Point all the CPUs in this policy to the same data */
+ for_each_cpu(cpu, pol->cpus)
+ per_cpu(powernow_data, cpu) = data;
+
+ return 0;
+
+err_out_exit_acpi:
+ powernow_k8_cpu_exit_acpi(data);
+
+err_out:
+ kfree(data);
+ return -ENODEV;
+}
+
+static int powernowk8_cpu_exit(struct cpufreq_policy *pol)
+{
+ struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu);
+ int cpu;
+
+ if (!data)
+ return -EINVAL;
+
+ powernow_k8_cpu_exit_acpi(data);
+
+ kfree(data->powernow_table);
+ kfree(data);
+ /* pol->cpus will be empty here, use related_cpus instead. */
+ for_each_cpu(cpu, pol->related_cpus)
+ per_cpu(powernow_data, cpu) = NULL;
+
+ return 0;
+}
+
+static void query_values_on_cpu(void *_err)
+{
+ int *err = _err;
+ struct powernow_k8_data *data = __this_cpu_read(powernow_data);
+
+ *err = query_current_values_with_pending_wait(data);
+}
+
+static unsigned int powernowk8_get(unsigned int cpu)
+{
+ struct powernow_k8_data *data = per_cpu(powernow_data, cpu);
+ unsigned int khz = 0;
+ int err;
+
+ if (!data)
+ return 0;
+
+ smp_call_function_single(cpu, query_values_on_cpu, &err, true);
+ if (err)
+ goto out;
+
+ khz = find_khz_freq_from_fid(data->currfid);
+
+
+out:
+ return khz;
+}
+
+static struct cpufreq_driver cpufreq_amd64_driver = {
+ .flags = CPUFREQ_ASYNC_NOTIFICATION,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = powernowk8_target,
+ .bios_limit = acpi_processor_get_bios_limit,
+ .init = powernowk8_cpu_init,
+ .exit = powernowk8_cpu_exit,
+ .get = powernowk8_get,
+ .name = "powernow-k8",
+ .attr = cpufreq_generic_attr,
+};
+
+static void __request_acpi_cpufreq(void)
+{
+ const char drv[] = "acpi-cpufreq";
+ const char *cur_drv;
+
+ cur_drv = cpufreq_get_current_driver();
+ if (!cur_drv)
+ goto request;
+
+ if (strncmp(cur_drv, drv, min_t(size_t, strlen(cur_drv), strlen(drv))))
+ pr_warn("WTF driver: %s\n", cur_drv);
+
+ return;
+
+ request:
+ pr_warn("This CPU is not supported anymore, using acpi-cpufreq instead.\n");
+ request_module(drv);
+}
+
+/* driver entry point for init */
+static int powernowk8_init(void)
+{
+ unsigned int i, supported_cpus = 0;
+ int ret;
+
+ if (!x86_match_cpu(powernow_k8_ids))
+ return -ENODEV;
+
+ if (boot_cpu_has(X86_FEATURE_HW_PSTATE)) {
+ __request_acpi_cpufreq();
+ return -ENODEV;
+ }
+
+ cpus_read_lock();
+ for_each_online_cpu(i) {
+ smp_call_function_single(i, check_supported_cpu, &ret, 1);
+ if (!ret)
+ supported_cpus++;
+ }
+
+ if (supported_cpus != num_online_cpus()) {
+ cpus_read_unlock();
+ return -ENODEV;
+ }
+ cpus_read_unlock();
+
+ ret = cpufreq_register_driver(&cpufreq_amd64_driver);
+ if (ret)
+ return ret;
+
+ pr_info("Found %d %s (%d cpu cores) (" VERSION ")\n",
+ num_online_nodes(), boot_cpu_data.x86_model_id, supported_cpus);
+
+ return ret;
+}
+
+/* driver entry point for term */
+static void __exit powernowk8_exit(void)
+{
+ pr_debug("exit\n");
+
+ cpufreq_unregister_driver(&cpufreq_amd64_driver);
+}
+
+MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com>");
+MODULE_AUTHOR("Mark Langsdorf <mark.langsdorf@amd.com>");
+MODULE_DESCRIPTION("AMD Athlon 64 and Opteron processor frequency driver.");
+MODULE_LICENSE("GPL");
+
+late_initcall(powernowk8_init);
+module_exit(powernowk8_exit);
diff --git a/drivers/cpufreq/powernow-k8.h b/drivers/cpufreq/powernow-k8.h
new file mode 100644
index 0000000000..83331ceb6b
--- /dev/null
+++ b/drivers/cpufreq/powernow-k8.h
@@ -0,0 +1,188 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * (c) 2003-2006 Advanced Micro Devices, Inc.
+ */
+
+struct powernow_k8_data {
+ unsigned int cpu;
+
+ u32 numps; /* number of p-states */
+ u32 batps; /* number of p-states supported on battery */
+
+ /* these values are constant when the PSB is used to determine
+ * vid/fid pairings, but are modified during the ->target() call
+ * when ACPI is used */
+ u32 rvo; /* ramp voltage offset */
+ u32 irt; /* isochronous relief time */
+ u32 vidmvs; /* usable value calculated from mvs */
+ u32 vstable; /* voltage stabilization time, units 20 us */
+ u32 plllock; /* pll lock time, units 1 us */
+ u32 exttype; /* extended interface = 1 */
+
+ /* keep track of the current fid / vid or pstate */
+ u32 currvid;
+ u32 currfid;
+
+ /* the powernow_table includes all frequency and vid/fid pairings:
+ * fid are the lower 8 bits of the index, vid are the upper 8 bits.
+ * frequency is in kHz */
+ struct cpufreq_frequency_table *powernow_table;
+
+ /* the acpi table needs to be kept. it's only available if ACPI was
+ * used to determine valid frequency/vid/fid states */
+ struct acpi_processor_performance acpi_data;
+
+ /* we need to keep track of associated cores, but let cpufreq
+ * handle hotplug events - so just point at cpufreq pol->cpus
+ * structure */
+ struct cpumask *available_cores;
+};
+
+/* processor's cpuid instruction support */
+#define CPUID_PROCESSOR_SIGNATURE 1 /* function 1 */
+#define CPUID_XFAM 0x0ff00000 /* extended family */
+#define CPUID_XFAM_K8 0
+#define CPUID_XMOD 0x000f0000 /* extended model */
+#define CPUID_XMOD_REV_MASK 0x000c0000
+#define CPUID_XFAM_10H 0x00100000 /* family 0x10 */
+#define CPUID_USE_XFAM_XMOD 0x00000f00
+#define CPUID_GET_MAX_CAPABILITIES 0x80000000
+#define CPUID_FREQ_VOLT_CAPABILITIES 0x80000007
+#define P_STATE_TRANSITION_CAPABLE 6
+
+/* Model Specific Registers for p-state transitions. MSRs are 64-bit. For */
+/* writes (wrmsr - opcode 0f 30), the register number is placed in ecx, and */
+/* the value to write is placed in edx:eax. For reads (rdmsr - opcode 0f 32), */
+/* the register number is placed in ecx, and the data is returned in edx:eax. */
+
+#define MSR_FIDVID_CTL 0xc0010041
+#define MSR_FIDVID_STATUS 0xc0010042
+
+/* Field definitions within the FID VID Low Control MSR : */
+#define MSR_C_LO_INIT_FID_VID 0x00010000
+#define MSR_C_LO_NEW_VID 0x00003f00
+#define MSR_C_LO_NEW_FID 0x0000003f
+#define MSR_C_LO_VID_SHIFT 8
+
+/* Field definitions within the FID VID High Control MSR : */
+#define MSR_C_HI_STP_GNT_TO 0x000fffff
+
+/* Field definitions within the FID VID Low Status MSR : */
+#define MSR_S_LO_CHANGE_PENDING 0x80000000 /* cleared when completed */
+#define MSR_S_LO_MAX_RAMP_VID 0x3f000000
+#define MSR_S_LO_MAX_FID 0x003f0000
+#define MSR_S_LO_START_FID 0x00003f00
+#define MSR_S_LO_CURRENT_FID 0x0000003f
+
+/* Field definitions within the FID VID High Status MSR : */
+#define MSR_S_HI_MIN_WORKING_VID 0x3f000000
+#define MSR_S_HI_MAX_WORKING_VID 0x003f0000
+#define MSR_S_HI_START_VID 0x00003f00
+#define MSR_S_HI_CURRENT_VID 0x0000003f
+#define MSR_C_HI_STP_GNT_BENIGN 0x00000001
+
+/*
+ * There are restrictions frequencies have to follow:
+ * - only 1 entry in the low fid table ( <=1.4GHz )
+ * - lowest entry in the high fid table must be >= 2 * the entry in the
+ * low fid table
+ * - lowest entry in the high fid table must be a <= 200MHz + 2 * the entry
+ * in the low fid table
+ * - the parts can only step at <= 200 MHz intervals, odd fid values are
+ * supported in revision G and later revisions.
+ * - lowest frequency must be >= interprocessor hypertransport link speed
+ * (only applies to MP systems obviously)
+ */
+
+/* fids (frequency identifiers) are arranged in 2 tables - lo and hi */
+#define LO_FID_TABLE_TOP 7 /* fid values marking the boundary */
+#define HI_FID_TABLE_BOTTOM 8 /* between the low and high tables */
+
+#define LO_VCOFREQ_TABLE_TOP 1400 /* corresponding vco frequency values */
+#define HI_VCOFREQ_TABLE_BOTTOM 1600
+
+#define MIN_FREQ_RESOLUTION 200 /* fids jump by 2 matching freq jumps by 200 */
+
+#define MAX_FID 0x2a /* Spec only gives FID values as far as 5 GHz */
+#define LEAST_VID 0x3e /* Lowest (numerically highest) useful vid value */
+
+#define MIN_FREQ 800 /* Min and max freqs, per spec */
+#define MAX_FREQ 5000
+
+#define INVALID_FID_MASK 0xffffffc0 /* not a valid fid if these bits are set */
+#define INVALID_VID_MASK 0xffffffc0 /* not a valid vid if these bits are set */
+
+#define VID_OFF 0x3f
+
+#define STOP_GRANT_5NS 1 /* min poss memory access latency for voltage change */
+
+#define PLL_LOCK_CONVERSION (1000/5) /* ms to ns, then divide by clock period */
+
+#define MAXIMUM_VID_STEPS 1 /* Current cpus only allow a single step of 25mV */
+#define VST_UNITS_20US 20 /* Voltage Stabilization Time is in units of 20us */
+
+/*
+ * Most values of interest are encoded in a single field of the _PSS
+ * entries: the "control" value.
+ */
+
+#define IRT_SHIFT 30
+#define RVO_SHIFT 28
+#define EXT_TYPE_SHIFT 27
+#define PLL_L_SHIFT 20
+#define MVS_SHIFT 18
+#define VST_SHIFT 11
+#define VID_SHIFT 6
+#define IRT_MASK 3
+#define RVO_MASK 3
+#define EXT_TYPE_MASK 1
+#define PLL_L_MASK 0x7f
+#define MVS_MASK 3
+#define VST_MASK 0x7f
+#define VID_MASK 0x1f
+#define FID_MASK 0x1f
+#define EXT_VID_MASK 0x3f
+#define EXT_FID_MASK 0x3f
+
+
+/*
+ * Version 1.4 of the PSB table. This table is constructed by BIOS and is
+ * to tell the OS's power management driver which VIDs and FIDs are
+ * supported by this particular processor.
+ * If the data in the PSB / PST is wrong, then this driver will program the
+ * wrong values into hardware, which is very likely to lead to a crash.
+ */
+
+#define PSB_ID_STRING "AMDK7PNOW!"
+#define PSB_ID_STRING_LEN 10
+
+#define PSB_VERSION_1_4 0x14
+
+struct psb_s {
+ u8 signature[10];
+ u8 tableversion;
+ u8 flags1;
+ u16 vstable;
+ u8 flags2;
+ u8 num_tables;
+ u32 cpuid;
+ u8 plllocktime;
+ u8 maxfid;
+ u8 maxvid;
+ u8 numps;
+};
+
+/* Pairs of fid/vid values are appended to the version 1.4 PSB table. */
+struct pst_s {
+ u8 fid;
+ u8 vid;
+};
+
+static int core_voltage_pre_transition(struct powernow_k8_data *data,
+ u32 reqvid, u32 regfid);
+static int core_voltage_post_transition(struct powernow_k8_data *data, u32 reqvid);
+static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid);
+
+static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index);
+
+static int fill_powernow_table_fidvid(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table);
diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c
new file mode 100644
index 0000000000..fddbd1ea16
--- /dev/null
+++ b/drivers/cpufreq/powernv-cpufreq.c
@@ -0,0 +1,1166 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * POWERNV cpufreq driver for the IBM POWER processors
+ *
+ * (C) Copyright IBM 2014
+ *
+ * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>
+ */
+
+#define pr_fmt(fmt) "powernv-cpufreq: " fmt
+
+#include <linux/kernel.h>
+#include <linux/sysfs.h>
+#include <linux/cpumask.h>
+#include <linux/module.h>
+#include <linux/cpufreq.h>
+#include <linux/smp.h>
+#include <linux/of.h>
+#include <linux/reboot.h>
+#include <linux/slab.h>
+#include <linux/cpu.h>
+#include <linux/hashtable.h>
+#include <trace/events/power.h>
+
+#include <asm/cputhreads.h>
+#include <asm/firmware.h>
+#include <asm/reg.h>
+#include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */
+#include <asm/opal.h>
+#include <linux/timer.h>
+
+#define POWERNV_MAX_PSTATES_ORDER 8
+#define POWERNV_MAX_PSTATES (1UL << (POWERNV_MAX_PSTATES_ORDER))
+#define PMSR_PSAFE_ENABLE (1UL << 30)
+#define PMSR_SPR_EM_DISABLE (1UL << 31)
+#define MAX_PSTATE_SHIFT 32
+#define LPSTATE_SHIFT 48
+#define GPSTATE_SHIFT 56
+#define MAX_NR_CHIPS 32
+
+#define MAX_RAMP_DOWN_TIME 5120
+/*
+ * On an idle system we want the global pstate to ramp-down from max value to
+ * min over a span of ~5 secs. Also we want it to initially ramp-down slowly and
+ * then ramp-down rapidly later on.
+ *
+ * This gives a percentage rampdown for time elapsed in milliseconds.
+ * ramp_down_percentage = ((ms * ms) >> 18)
+ * ~= 3.8 * (sec * sec)
+ *
+ * At 0 ms ramp_down_percent = 0
+ * At 5120 ms ramp_down_percent = 100
+ */
+#define ramp_down_percent(time) ((time * time) >> 18)
+
+/* Interval after which the timer is queued to bring down global pstate */
+#define GPSTATE_TIMER_INTERVAL 2000
+
+/**
+ * struct global_pstate_info - Per policy data structure to maintain history of
+ * global pstates
+ * @highest_lpstate_idx: The local pstate index from which we are
+ * ramping down
+ * @elapsed_time: Time in ms spent in ramping down from
+ * highest_lpstate_idx
+ * @last_sampled_time: Time from boot in ms when global pstates were
+ * last set
+ * @last_lpstate_idx: Last set value of local pstate and global
+ * @last_gpstate_idx: pstate in terms of cpufreq table index
+ * @timer: Is used for ramping down if cpu goes idle for
+ * a long time with global pstate held high
+ * @gpstate_lock: A spinlock to maintain synchronization between
+ * routines called by the timer handler and
+ * governer's target_index calls
+ * @policy: Associated CPUFreq policy
+ */
+struct global_pstate_info {
+ int highest_lpstate_idx;
+ unsigned int elapsed_time;
+ unsigned int last_sampled_time;
+ int last_lpstate_idx;
+ int last_gpstate_idx;
+ spinlock_t gpstate_lock;
+ struct timer_list timer;
+ struct cpufreq_policy *policy;
+};
+
+static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
+
+static DEFINE_HASHTABLE(pstate_revmap, POWERNV_MAX_PSTATES_ORDER);
+/**
+ * struct pstate_idx_revmap_data: Entry in the hashmap pstate_revmap
+ * indexed by a function of pstate id.
+ *
+ * @pstate_id: pstate id for this entry.
+ *
+ * @cpufreq_table_idx: Index into the powernv_freqs
+ * cpufreq_frequency_table for frequency
+ * corresponding to pstate_id.
+ *
+ * @hentry: hlist_node that hooks this entry into the pstate_revmap
+ * hashtable
+ */
+struct pstate_idx_revmap_data {
+ u8 pstate_id;
+ unsigned int cpufreq_table_idx;
+ struct hlist_node hentry;
+};
+
+static bool rebooting, throttled, occ_reset;
+
+static const char * const throttle_reason[] = {
+ "No throttling",
+ "Power Cap",
+ "Processor Over Temperature",
+ "Power Supply Failure",
+ "Over Current",
+ "OCC Reset"
+};
+
+enum throttle_reason_type {
+ NO_THROTTLE = 0,
+ POWERCAP,
+ CPU_OVERTEMP,
+ POWER_SUPPLY_FAILURE,
+ OVERCURRENT,
+ OCC_RESET_THROTTLE,
+ OCC_MAX_REASON
+};
+
+static struct chip {
+ unsigned int id;
+ bool throttled;
+ bool restore;
+ u8 throttle_reason;
+ cpumask_t mask;
+ struct work_struct throttle;
+ int throttle_turbo;
+ int throttle_sub_turbo;
+ int reason[OCC_MAX_REASON];
+} *chips;
+
+static int nr_chips;
+static DEFINE_PER_CPU(struct chip *, chip_info);
+
+/*
+ * Note:
+ * The set of pstates consists of contiguous integers.
+ * powernv_pstate_info stores the index of the frequency table for
+ * max, min and nominal frequencies. It also stores number of
+ * available frequencies.
+ *
+ * powernv_pstate_info.nominal indicates the index to the highest
+ * non-turbo frequency.
+ */
+static struct powernv_pstate_info {
+ unsigned int min;
+ unsigned int max;
+ unsigned int nominal;
+ unsigned int nr_pstates;
+ bool wof_enabled;
+} powernv_pstate_info;
+
+static inline u8 extract_pstate(u64 pmsr_val, unsigned int shift)
+{
+ return ((pmsr_val >> shift) & 0xFF);
+}
+
+#define extract_local_pstate(x) extract_pstate(x, LPSTATE_SHIFT)
+#define extract_global_pstate(x) extract_pstate(x, GPSTATE_SHIFT)
+#define extract_max_pstate(x) extract_pstate(x, MAX_PSTATE_SHIFT)
+
+/* Use following functions for conversions between pstate_id and index */
+
+/*
+ * idx_to_pstate : Returns the pstate id corresponding to the
+ * frequency in the cpufreq frequency table
+ * powernv_freqs indexed by @i.
+ *
+ * If @i is out of bound, this will return the pstate
+ * corresponding to the nominal frequency.
+ */
+static inline u8 idx_to_pstate(unsigned int i)
+{
+ if (unlikely(i >= powernv_pstate_info.nr_pstates)) {
+ pr_warn_once("idx_to_pstate: index %u is out of bound\n", i);
+ return powernv_freqs[powernv_pstate_info.nominal].driver_data;
+ }
+
+ return powernv_freqs[i].driver_data;
+}
+
+/*
+ * pstate_to_idx : Returns the index in the cpufreq frequencytable
+ * powernv_freqs for the frequency whose corresponding
+ * pstate id is @pstate.
+ *
+ * If no frequency corresponding to @pstate is found,
+ * this will return the index of the nominal
+ * frequency.
+ */
+static unsigned int pstate_to_idx(u8 pstate)
+{
+ unsigned int key = pstate % POWERNV_MAX_PSTATES;
+ struct pstate_idx_revmap_data *revmap_data;
+
+ hash_for_each_possible(pstate_revmap, revmap_data, hentry, key) {
+ if (revmap_data->pstate_id == pstate)
+ return revmap_data->cpufreq_table_idx;
+ }
+
+ pr_warn_once("pstate_to_idx: pstate 0x%x not found\n", pstate);
+ return powernv_pstate_info.nominal;
+}
+
+static inline void reset_gpstates(struct cpufreq_policy *policy)
+{
+ struct global_pstate_info *gpstates = policy->driver_data;
+
+ gpstates->highest_lpstate_idx = 0;
+ gpstates->elapsed_time = 0;
+ gpstates->last_sampled_time = 0;
+ gpstates->last_lpstate_idx = 0;
+ gpstates->last_gpstate_idx = 0;
+}
+
+/*
+ * Initialize the freq table based on data obtained
+ * from the firmware passed via device-tree
+ */
+static int init_powernv_pstates(void)
+{
+ struct device_node *power_mgt;
+ int i, nr_pstates = 0;
+ const __be32 *pstate_ids, *pstate_freqs;
+ u32 len_ids, len_freqs;
+ u32 pstate_min, pstate_max, pstate_nominal;
+ u32 pstate_turbo, pstate_ultra_turbo;
+ int rc = -ENODEV;
+
+ power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
+ if (!power_mgt) {
+ pr_warn("power-mgt node not found\n");
+ return -ENODEV;
+ }
+
+ if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) {
+ pr_warn("ibm,pstate-min node not found\n");
+ goto out;
+ }
+
+ if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) {
+ pr_warn("ibm,pstate-max node not found\n");
+ goto out;
+ }
+
+ if (of_property_read_u32(power_mgt, "ibm,pstate-nominal",
+ &pstate_nominal)) {
+ pr_warn("ibm,pstate-nominal not found\n");
+ goto out;
+ }
+
+ if (of_property_read_u32(power_mgt, "ibm,pstate-ultra-turbo",
+ &pstate_ultra_turbo)) {
+ powernv_pstate_info.wof_enabled = false;
+ goto next;
+ }
+
+ if (of_property_read_u32(power_mgt, "ibm,pstate-turbo",
+ &pstate_turbo)) {
+ powernv_pstate_info.wof_enabled = false;
+ goto next;
+ }
+
+ if (pstate_turbo == pstate_ultra_turbo)
+ powernv_pstate_info.wof_enabled = false;
+ else
+ powernv_pstate_info.wof_enabled = true;
+
+next:
+ pr_info("cpufreq pstate min 0x%x nominal 0x%x max 0x%x\n", pstate_min,
+ pstate_nominal, pstate_max);
+ pr_info("Workload Optimized Frequency is %s in the platform\n",
+ (powernv_pstate_info.wof_enabled) ? "enabled" : "disabled");
+
+ pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids);
+ if (!pstate_ids) {
+ pr_warn("ibm,pstate-ids not found\n");
+ goto out;
+ }
+
+ pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz",
+ &len_freqs);
+ if (!pstate_freqs) {
+ pr_warn("ibm,pstate-frequencies-mhz not found\n");
+ goto out;
+ }
+
+ if (len_ids != len_freqs) {
+ pr_warn("Entries in ibm,pstate-ids and "
+ "ibm,pstate-frequencies-mhz does not match\n");
+ }
+
+ nr_pstates = min(len_ids, len_freqs) / sizeof(u32);
+ if (!nr_pstates) {
+ pr_warn("No PStates found\n");
+ goto out;
+ }
+
+ powernv_pstate_info.nr_pstates = nr_pstates;
+ pr_debug("NR PStates %d\n", nr_pstates);
+
+ for (i = 0; i < nr_pstates; i++) {
+ u32 id = be32_to_cpu(pstate_ids[i]);
+ u32 freq = be32_to_cpu(pstate_freqs[i]);
+ struct pstate_idx_revmap_data *revmap_data;
+ unsigned int key;
+
+ pr_debug("PState id %d freq %d MHz\n", id, freq);
+ powernv_freqs[i].frequency = freq * 1000; /* kHz */
+ powernv_freqs[i].driver_data = id & 0xFF;
+
+ revmap_data = kmalloc(sizeof(*revmap_data), GFP_KERNEL);
+ if (!revmap_data) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ revmap_data->pstate_id = id & 0xFF;
+ revmap_data->cpufreq_table_idx = i;
+ key = (revmap_data->pstate_id) % POWERNV_MAX_PSTATES;
+ hash_add(pstate_revmap, &revmap_data->hentry, key);
+
+ if (id == pstate_max)
+ powernv_pstate_info.max = i;
+ if (id == pstate_nominal)
+ powernv_pstate_info.nominal = i;
+ if (id == pstate_min)
+ powernv_pstate_info.min = i;
+
+ if (powernv_pstate_info.wof_enabled && id == pstate_turbo) {
+ int j;
+
+ for (j = i - 1; j >= (int)powernv_pstate_info.max; j--)
+ powernv_freqs[j].flags = CPUFREQ_BOOST_FREQ;
+ }
+ }
+
+ /* End of list marker entry */
+ powernv_freqs[i].frequency = CPUFREQ_TABLE_END;
+
+ of_node_put(power_mgt);
+ return 0;
+out:
+ of_node_put(power_mgt);
+ return rc;
+}
+
+/* Returns the CPU frequency corresponding to the pstate_id. */
+static unsigned int pstate_id_to_freq(u8 pstate_id)
+{
+ int i;
+
+ i = pstate_to_idx(pstate_id);
+ if (i >= powernv_pstate_info.nr_pstates || i < 0) {
+ pr_warn("PState id 0x%x outside of PState table, reporting nominal id 0x%x instead\n",
+ pstate_id, idx_to_pstate(powernv_pstate_info.nominal));
+ i = powernv_pstate_info.nominal;
+ }
+
+ return powernv_freqs[i].frequency;
+}
+
+/*
+ * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by
+ * the firmware
+ */
+static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,
+ char *buf)
+{
+ return sprintf(buf, "%u\n",
+ powernv_freqs[powernv_pstate_info.nominal].frequency);
+}
+
+static struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =
+ __ATTR_RO(cpuinfo_nominal_freq);
+
+#define SCALING_BOOST_FREQS_ATTR_INDEX 2
+
+static struct freq_attr *powernv_cpu_freq_attr[] = {
+ &cpufreq_freq_attr_scaling_available_freqs,
+ &cpufreq_freq_attr_cpuinfo_nominal_freq,
+ &cpufreq_freq_attr_scaling_boost_freqs,
+ NULL,
+};
+
+#define throttle_attr(name, member) \
+static ssize_t name##_show(struct cpufreq_policy *policy, char *buf) \
+{ \
+ struct chip *chip = per_cpu(chip_info, policy->cpu); \
+ \
+ return sprintf(buf, "%u\n", chip->member); \
+} \
+ \
+static struct freq_attr throttle_attr_##name = __ATTR_RO(name) \
+
+throttle_attr(unthrottle, reason[NO_THROTTLE]);
+throttle_attr(powercap, reason[POWERCAP]);
+throttle_attr(overtemp, reason[CPU_OVERTEMP]);
+throttle_attr(supply_fault, reason[POWER_SUPPLY_FAILURE]);
+throttle_attr(overcurrent, reason[OVERCURRENT]);
+throttle_attr(occ_reset, reason[OCC_RESET_THROTTLE]);
+throttle_attr(turbo_stat, throttle_turbo);
+throttle_attr(sub_turbo_stat, throttle_sub_turbo);
+
+static struct attribute *throttle_attrs[] = {
+ &throttle_attr_unthrottle.attr,
+ &throttle_attr_powercap.attr,
+ &throttle_attr_overtemp.attr,
+ &throttle_attr_supply_fault.attr,
+ &throttle_attr_overcurrent.attr,
+ &throttle_attr_occ_reset.attr,
+ &throttle_attr_turbo_stat.attr,
+ &throttle_attr_sub_turbo_stat.attr,
+ NULL,
+};
+
+static const struct attribute_group throttle_attr_grp = {
+ .name = "throttle_stats",
+ .attrs = throttle_attrs,
+};
+
+/* Helper routines */
+
+/* Access helpers to power mgt SPR */
+
+static inline unsigned long get_pmspr(unsigned long sprn)
+{
+ switch (sprn) {
+ case SPRN_PMCR:
+ return mfspr(SPRN_PMCR);
+
+ case SPRN_PMICR:
+ return mfspr(SPRN_PMICR);
+
+ case SPRN_PMSR:
+ return mfspr(SPRN_PMSR);
+ }
+ BUG();
+}
+
+static inline void set_pmspr(unsigned long sprn, unsigned long val)
+{
+ switch (sprn) {
+ case SPRN_PMCR:
+ mtspr(SPRN_PMCR, val);
+ return;
+
+ case SPRN_PMICR:
+ mtspr(SPRN_PMICR, val);
+ return;
+ }
+ BUG();
+}
+
+/*
+ * Use objects of this type to query/update
+ * pstates on a remote CPU via smp_call_function.
+ */
+struct powernv_smp_call_data {
+ unsigned int freq;
+ u8 pstate_id;
+ u8 gpstate_id;
+};
+
+/*
+ * powernv_read_cpu_freq: Reads the current frequency on this CPU.
+ *
+ * Called via smp_call_function.
+ *
+ * Note: The caller of the smp_call_function should pass an argument of
+ * the type 'struct powernv_smp_call_data *' along with this function.
+ *
+ * The current frequency on this CPU will be returned via
+ * ((struct powernv_smp_call_data *)arg)->freq;
+ */
+static void powernv_read_cpu_freq(void *arg)
+{
+ unsigned long pmspr_val;
+ struct powernv_smp_call_data *freq_data = arg;
+
+ pmspr_val = get_pmspr(SPRN_PMSR);
+ freq_data->pstate_id = extract_local_pstate(pmspr_val);
+ freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);
+
+ pr_debug("cpu %d pmsr %016lX pstate_id 0x%x frequency %d kHz\n",
+ raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
+ freq_data->freq);
+}
+
+/*
+ * powernv_cpufreq_get: Returns the CPU frequency as reported by the
+ * firmware for CPU 'cpu'. This value is reported through the sysfs
+ * file cpuinfo_cur_freq.
+ */
+static unsigned int powernv_cpufreq_get(unsigned int cpu)
+{
+ struct powernv_smp_call_data freq_data;
+
+ smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq,
+ &freq_data, 1);
+
+ return freq_data.freq;
+}
+
+/*
+ * set_pstate: Sets the pstate on this CPU.
+ *
+ * This is called via an smp_call_function.
+ *
+ * The caller must ensure that freq_data is of the type
+ * (struct powernv_smp_call_data *) and the pstate_id which needs to be set
+ * on this CPU should be present in freq_data->pstate_id.
+ */
+static void set_pstate(void *data)
+{
+ unsigned long val;
+ struct powernv_smp_call_data *freq_data = data;
+ unsigned long pstate_ul = freq_data->pstate_id;
+ unsigned long gpstate_ul = freq_data->gpstate_id;
+
+ val = get_pmspr(SPRN_PMCR);
+ val = val & 0x0000FFFFFFFFFFFFULL;
+
+ pstate_ul = pstate_ul & 0xFF;
+ gpstate_ul = gpstate_ul & 0xFF;
+
+ /* Set both global(bits 56..63) and local(bits 48..55) PStates */
+ val = val | (gpstate_ul << 56) | (pstate_ul << 48);
+
+ pr_debug("Setting cpu %d pmcr to %016lX\n",
+ raw_smp_processor_id(), val);
+ set_pmspr(SPRN_PMCR, val);
+}
+
+/*
+ * get_nominal_index: Returns the index corresponding to the nominal
+ * pstate in the cpufreq table
+ */
+static inline unsigned int get_nominal_index(void)
+{
+ return powernv_pstate_info.nominal;
+}
+
+static void powernv_cpufreq_throttle_check(void *data)
+{
+ struct chip *chip;
+ unsigned int cpu = smp_processor_id();
+ unsigned long pmsr;
+ u8 pmsr_pmax;
+ unsigned int pmsr_pmax_idx;
+
+ pmsr = get_pmspr(SPRN_PMSR);
+ chip = this_cpu_read(chip_info);
+
+ /* Check for Pmax Capping */
+ pmsr_pmax = extract_max_pstate(pmsr);
+ pmsr_pmax_idx = pstate_to_idx(pmsr_pmax);
+ if (pmsr_pmax_idx != powernv_pstate_info.max) {
+ if (chip->throttled)
+ goto next;
+ chip->throttled = true;
+ if (pmsr_pmax_idx > powernv_pstate_info.nominal) {
+ pr_warn_once("CPU %d on Chip %u has Pmax(0x%x) reduced below that of nominal frequency(0x%x)\n",
+ cpu, chip->id, pmsr_pmax,
+ idx_to_pstate(powernv_pstate_info.nominal));
+ chip->throttle_sub_turbo++;
+ } else {
+ chip->throttle_turbo++;
+ }
+ trace_powernv_throttle(chip->id,
+ throttle_reason[chip->throttle_reason],
+ pmsr_pmax);
+ } else if (chip->throttled) {
+ chip->throttled = false;
+ trace_powernv_throttle(chip->id,
+ throttle_reason[chip->throttle_reason],
+ pmsr_pmax);
+ }
+
+ /* Check if Psafe_mode_active is set in PMSR. */
+next:
+ if (pmsr & PMSR_PSAFE_ENABLE) {
+ throttled = true;
+ pr_info("Pstate set to safe frequency\n");
+ }
+
+ /* Check if SPR_EM_DISABLE is set in PMSR */
+ if (pmsr & PMSR_SPR_EM_DISABLE) {
+ throttled = true;
+ pr_info("Frequency Control disabled from OS\n");
+ }
+
+ if (throttled) {
+ pr_info("PMSR = %16lx\n", pmsr);
+ pr_warn("CPU Frequency could be throttled\n");
+ }
+}
+
+/**
+ * calc_global_pstate - Calculate global pstate
+ * @elapsed_time: Elapsed time in milliseconds
+ * @local_pstate_idx: New local pstate
+ * @highest_lpstate_idx: pstate from which its ramping down
+ *
+ * Finds the appropriate global pstate based on the pstate from which its
+ * ramping down and the time elapsed in ramping down. It follows a quadratic
+ * equation which ensures that it reaches ramping down to pmin in 5sec.
+ */
+static inline int calc_global_pstate(unsigned int elapsed_time,
+ int highest_lpstate_idx,
+ int local_pstate_idx)
+{
+ int index_diff;
+
+ /*
+ * Using ramp_down_percent we get the percentage of rampdown
+ * that we are expecting to be dropping. Difference between
+ * highest_lpstate_idx and powernv_pstate_info.min will give a absolute
+ * number of how many pstates we will drop eventually by the end of
+ * 5 seconds, then just scale it get the number pstates to be dropped.
+ */
+ index_diff = ((int)ramp_down_percent(elapsed_time) *
+ (powernv_pstate_info.min - highest_lpstate_idx)) / 100;
+
+ /* Ensure that global pstate is >= to local pstate */
+ if (highest_lpstate_idx + index_diff >= local_pstate_idx)
+ return local_pstate_idx;
+ else
+ return highest_lpstate_idx + index_diff;
+}
+
+static inline void queue_gpstate_timer(struct global_pstate_info *gpstates)
+{
+ unsigned int timer_interval;
+
+ /*
+ * Setting up timer to fire after GPSTATE_TIMER_INTERVAL ms, But
+ * if it exceeds MAX_RAMP_DOWN_TIME ms for ramp down time.
+ * Set timer such that it fires exactly at MAX_RAMP_DOWN_TIME
+ * seconds of ramp down time.
+ */
+ if ((gpstates->elapsed_time + GPSTATE_TIMER_INTERVAL)
+ > MAX_RAMP_DOWN_TIME)
+ timer_interval = MAX_RAMP_DOWN_TIME - gpstates->elapsed_time;
+ else
+ timer_interval = GPSTATE_TIMER_INTERVAL;
+
+ mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval));
+}
+
+/**
+ * gpstate_timer_handler
+ *
+ * @t: Timer context used to fetch global pstate info struct
+ *
+ * This handler brings down the global pstate closer to the local pstate
+ * according quadratic equation. Queues a new timer if it is still not equal
+ * to local pstate
+ */
+static void gpstate_timer_handler(struct timer_list *t)
+{
+ struct global_pstate_info *gpstates = from_timer(gpstates, t, timer);
+ struct cpufreq_policy *policy = gpstates->policy;
+ int gpstate_idx, lpstate_idx;
+ unsigned long val;
+ unsigned int time_diff = jiffies_to_msecs(jiffies)
+ - gpstates->last_sampled_time;
+ struct powernv_smp_call_data freq_data;
+
+ if (!spin_trylock(&gpstates->gpstate_lock))
+ return;
+ /*
+ * If the timer has migrated to the different cpu then bring
+ * it back to one of the policy->cpus
+ */
+ if (!cpumask_test_cpu(raw_smp_processor_id(), policy->cpus)) {
+ gpstates->timer.expires = jiffies + msecs_to_jiffies(1);
+ add_timer_on(&gpstates->timer, cpumask_first(policy->cpus));
+ spin_unlock(&gpstates->gpstate_lock);
+ return;
+ }
+
+ /*
+ * If PMCR was last updated was using fast_swtich then
+ * We may have wrong in gpstate->last_lpstate_idx
+ * value. Hence, read from PMCR to get correct data.
+ */
+ val = get_pmspr(SPRN_PMCR);
+ freq_data.gpstate_id = extract_global_pstate(val);
+ freq_data.pstate_id = extract_local_pstate(val);
+ if (freq_data.gpstate_id == freq_data.pstate_id) {
+ reset_gpstates(policy);
+ spin_unlock(&gpstates->gpstate_lock);
+ return;
+ }
+
+ gpstates->last_sampled_time += time_diff;
+ gpstates->elapsed_time += time_diff;
+
+ if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
+ gpstate_idx = pstate_to_idx(freq_data.pstate_id);
+ lpstate_idx = gpstate_idx;
+ reset_gpstates(policy);
+ gpstates->highest_lpstate_idx = gpstate_idx;
+ } else {
+ lpstate_idx = pstate_to_idx(freq_data.pstate_id);
+ gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
+ gpstates->highest_lpstate_idx,
+ lpstate_idx);
+ }
+ freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
+ gpstates->last_gpstate_idx = gpstate_idx;
+ gpstates->last_lpstate_idx = lpstate_idx;
+ /*
+ * If local pstate is equal to global pstate, rampdown is over
+ * So timer is not required to be queued.
+ */
+ if (gpstate_idx != gpstates->last_lpstate_idx)
+ queue_gpstate_timer(gpstates);
+
+ set_pstate(&freq_data);
+ spin_unlock(&gpstates->gpstate_lock);
+}
+
+/*
+ * powernv_cpufreq_target_index: Sets the frequency corresponding to
+ * the cpufreq table entry indexed by new_index on the cpus in the
+ * mask policy->cpus
+ */
+static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
+ unsigned int new_index)
+{
+ struct powernv_smp_call_data freq_data;
+ unsigned int cur_msec, gpstate_idx;
+ struct global_pstate_info *gpstates = policy->driver_data;
+
+ if (unlikely(rebooting) && new_index != get_nominal_index())
+ return 0;
+
+ if (!throttled) {
+ /* we don't want to be preempted while
+ * checking if the CPU frequency has been throttled
+ */
+ preempt_disable();
+ powernv_cpufreq_throttle_check(NULL);
+ preempt_enable();
+ }
+
+ cur_msec = jiffies_to_msecs(get_jiffies_64());
+
+ freq_data.pstate_id = idx_to_pstate(new_index);
+ if (!gpstates) {
+ freq_data.gpstate_id = freq_data.pstate_id;
+ goto no_gpstate;
+ }
+
+ spin_lock(&gpstates->gpstate_lock);
+
+ if (!gpstates->last_sampled_time) {
+ gpstate_idx = new_index;
+ gpstates->highest_lpstate_idx = new_index;
+ goto gpstates_done;
+ }
+
+ if (gpstates->last_gpstate_idx < new_index) {
+ gpstates->elapsed_time += cur_msec -
+ gpstates->last_sampled_time;
+
+ /*
+ * If its has been ramping down for more than MAX_RAMP_DOWN_TIME
+ * we should be resetting all global pstate related data. Set it
+ * equal to local pstate to start fresh.
+ */
+ if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
+ reset_gpstates(policy);
+ gpstates->highest_lpstate_idx = new_index;
+ gpstate_idx = new_index;
+ } else {
+ /* Elaspsed_time is less than 5 seconds, continue to rampdown */
+ gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
+ gpstates->highest_lpstate_idx,
+ new_index);
+ }
+ } else {
+ reset_gpstates(policy);
+ gpstates->highest_lpstate_idx = new_index;
+ gpstate_idx = new_index;
+ }
+
+ /*
+ * If local pstate is equal to global pstate, rampdown is over
+ * So timer is not required to be queued.
+ */
+ if (gpstate_idx != new_index)
+ queue_gpstate_timer(gpstates);
+ else
+ del_timer_sync(&gpstates->timer);
+
+gpstates_done:
+ freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
+ gpstates->last_sampled_time = cur_msec;
+ gpstates->last_gpstate_idx = gpstate_idx;
+ gpstates->last_lpstate_idx = new_index;
+
+ spin_unlock(&gpstates->gpstate_lock);
+
+no_gpstate:
+ /*
+ * Use smp_call_function to send IPI and execute the
+ * mtspr on target CPU. We could do that without IPI
+ * if current CPU is within policy->cpus (core)
+ */
+ smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
+ return 0;
+}
+
+static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ int base, i;
+ struct kernfs_node *kn;
+ struct global_pstate_info *gpstates;
+
+ base = cpu_first_thread_sibling(policy->cpu);
+
+ for (i = 0; i < threads_per_core; i++)
+ cpumask_set_cpu(base + i, policy->cpus);
+
+ kn = kernfs_find_and_get(policy->kobj.sd, throttle_attr_grp.name);
+ if (!kn) {
+ int ret;
+
+ ret = sysfs_create_group(&policy->kobj, &throttle_attr_grp);
+ if (ret) {
+ pr_info("Failed to create throttle stats directory for cpu %d\n",
+ policy->cpu);
+ return ret;
+ }
+ } else {
+ kernfs_put(kn);
+ }
+
+ policy->freq_table = powernv_freqs;
+ policy->fast_switch_possible = true;
+
+ if (pvr_version_is(PVR_POWER9))
+ return 0;
+
+ /* Initialise Gpstate ramp-down timer only on POWER8 */
+ gpstates = kzalloc(sizeof(*gpstates), GFP_KERNEL);
+ if (!gpstates)
+ return -ENOMEM;
+
+ policy->driver_data = gpstates;
+
+ /* initialize timer */
+ gpstates->policy = policy;
+ timer_setup(&gpstates->timer, gpstate_timer_handler,
+ TIMER_PINNED | TIMER_DEFERRABLE);
+ gpstates->timer.expires = jiffies +
+ msecs_to_jiffies(GPSTATE_TIMER_INTERVAL);
+ spin_lock_init(&gpstates->gpstate_lock);
+
+ return 0;
+}
+
+static int powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct powernv_smp_call_data freq_data;
+ struct global_pstate_info *gpstates = policy->driver_data;
+
+ freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min);
+ freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min);
+ smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);
+ if (gpstates)
+ del_timer_sync(&gpstates->timer);
+
+ kfree(policy->driver_data);
+
+ return 0;
+}
+
+static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb,
+ unsigned long action, void *unused)
+{
+ int cpu;
+ struct cpufreq_policy *cpu_policy;
+
+ rebooting = true;
+ for_each_online_cpu(cpu) {
+ cpu_policy = cpufreq_cpu_get(cpu);
+ if (!cpu_policy)
+ continue;
+ powernv_cpufreq_target_index(cpu_policy, get_nominal_index());
+ cpufreq_cpu_put(cpu_policy);
+ }
+
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block powernv_cpufreq_reboot_nb = {
+ .notifier_call = powernv_cpufreq_reboot_notifier,
+};
+
+static void powernv_cpufreq_work_fn(struct work_struct *work)
+{
+ struct chip *chip = container_of(work, struct chip, throttle);
+ struct cpufreq_policy *policy;
+ unsigned int cpu;
+ cpumask_t mask;
+
+ cpus_read_lock();
+ cpumask_and(&mask, &chip->mask, cpu_online_mask);
+ smp_call_function_any(&mask,
+ powernv_cpufreq_throttle_check, NULL, 0);
+
+ if (!chip->restore)
+ goto out;
+
+ chip->restore = false;
+ for_each_cpu(cpu, &mask) {
+ int index;
+
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy)
+ continue;
+ index = cpufreq_table_find_index_c(policy, policy->cur, false);
+ powernv_cpufreq_target_index(policy, index);
+ cpumask_andnot(&mask, &mask, policy->cpus);
+ cpufreq_cpu_put(policy);
+ }
+out:
+ cpus_read_unlock();
+}
+
+static int powernv_cpufreq_occ_msg(struct notifier_block *nb,
+ unsigned long msg_type, void *_msg)
+{
+ struct opal_msg *msg = _msg;
+ struct opal_occ_msg omsg;
+ int i;
+
+ if (msg_type != OPAL_MSG_OCC)
+ return 0;
+
+ omsg.type = be64_to_cpu(msg->params[0]);
+
+ switch (omsg.type) {
+ case OCC_RESET:
+ occ_reset = true;
+ pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");
+ /*
+ * powernv_cpufreq_throttle_check() is called in
+ * target() callback which can detect the throttle state
+ * for governors like ondemand.
+ * But static governors will not call target() often thus
+ * report throttling here.
+ */
+ if (!throttled) {
+ throttled = true;
+ pr_warn("CPU frequency is throttled for duration\n");
+ }
+
+ break;
+ case OCC_LOAD:
+ pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");
+ break;
+ case OCC_THROTTLE:
+ omsg.chip = be64_to_cpu(msg->params[1]);
+ omsg.throttle_status = be64_to_cpu(msg->params[2]);
+
+ if (occ_reset) {
+ occ_reset = false;
+ throttled = false;
+ pr_info("OCC Active, CPU frequency is no longer throttled\n");
+
+ for (i = 0; i < nr_chips; i++) {
+ chips[i].restore = true;
+ schedule_work(&chips[i].throttle);
+ }
+
+ return 0;
+ }
+
+ for (i = 0; i < nr_chips; i++)
+ if (chips[i].id == omsg.chip)
+ break;
+
+ if (omsg.throttle_status >= 0 &&
+ omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) {
+ chips[i].throttle_reason = omsg.throttle_status;
+ chips[i].reason[omsg.throttle_status]++;
+ }
+
+ if (!omsg.throttle_status)
+ chips[i].restore = true;
+
+ schedule_work(&chips[i].throttle);
+ }
+ return 0;
+}
+
+static struct notifier_block powernv_cpufreq_opal_nb = {
+ .notifier_call = powernv_cpufreq_occ_msg,
+ .next = NULL,
+ .priority = 0,
+};
+
+static unsigned int powernv_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ int index;
+ struct powernv_smp_call_data freq_data;
+
+ index = cpufreq_table_find_index_dl(policy, target_freq, false);
+ freq_data.pstate_id = powernv_freqs[index].driver_data;
+ freq_data.gpstate_id = powernv_freqs[index].driver_data;
+ set_pstate(&freq_data);
+
+ return powernv_freqs[index].frequency;
+}
+
+static struct cpufreq_driver powernv_cpufreq_driver = {
+ .name = "powernv-cpufreq",
+ .flags = CPUFREQ_CONST_LOOPS,
+ .init = powernv_cpufreq_cpu_init,
+ .exit = powernv_cpufreq_cpu_exit,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = powernv_cpufreq_target_index,
+ .fast_switch = powernv_fast_switch,
+ .get = powernv_cpufreq_get,
+ .attr = powernv_cpu_freq_attr,
+};
+
+static int init_chip_info(void)
+{
+ unsigned int *chip;
+ unsigned int cpu, i;
+ unsigned int prev_chip_id = UINT_MAX;
+ cpumask_t *chip_cpu_mask;
+ int ret = 0;
+
+ chip = kcalloc(num_possible_cpus(), sizeof(*chip), GFP_KERNEL);
+ if (!chip)
+ return -ENOMEM;
+
+ /* Allocate a chip cpu mask large enough to fit mask for all chips */
+ chip_cpu_mask = kcalloc(MAX_NR_CHIPS, sizeof(cpumask_t), GFP_KERNEL);
+ if (!chip_cpu_mask) {
+ ret = -ENOMEM;
+ goto free_and_return;
+ }
+
+ for_each_possible_cpu(cpu) {
+ unsigned int id = cpu_to_chip_id(cpu);
+
+ if (prev_chip_id != id) {
+ prev_chip_id = id;
+ chip[nr_chips++] = id;
+ }
+ cpumask_set_cpu(cpu, &chip_cpu_mask[nr_chips-1]);
+ }
+
+ chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL);
+ if (!chips) {
+ ret = -ENOMEM;
+ goto out_free_chip_cpu_mask;
+ }
+
+ for (i = 0; i < nr_chips; i++) {
+ chips[i].id = chip[i];
+ cpumask_copy(&chips[i].mask, &chip_cpu_mask[i]);
+ INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);
+ for_each_cpu(cpu, &chips[i].mask)
+ per_cpu(chip_info, cpu) = &chips[i];
+ }
+
+out_free_chip_cpu_mask:
+ kfree(chip_cpu_mask);
+free_and_return:
+ kfree(chip);
+ return ret;
+}
+
+static inline void clean_chip_info(void)
+{
+ int i;
+
+ /* flush any pending work items */
+ if (chips)
+ for (i = 0; i < nr_chips; i++)
+ cancel_work_sync(&chips[i].throttle);
+ kfree(chips);
+}
+
+static inline void unregister_all_notifiers(void)
+{
+ opal_message_notifier_unregister(OPAL_MSG_OCC,
+ &powernv_cpufreq_opal_nb);
+ unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);
+}
+
+static int __init powernv_cpufreq_init(void)
+{
+ int rc = 0;
+
+ /* Don't probe on pseries (guest) platforms */
+ if (!firmware_has_feature(FW_FEATURE_OPAL))
+ return -ENODEV;
+
+ /* Discover pstates from device tree and init */
+ rc = init_powernv_pstates();
+ if (rc)
+ goto out;
+
+ /* Populate chip info */
+ rc = init_chip_info();
+ if (rc)
+ goto out;
+
+ if (powernv_pstate_info.wof_enabled)
+ powernv_cpufreq_driver.boost_enabled = true;
+ else
+ powernv_cpu_freq_attr[SCALING_BOOST_FREQS_ATTR_INDEX] = NULL;
+
+ rc = cpufreq_register_driver(&powernv_cpufreq_driver);
+ if (rc) {
+ pr_info("Failed to register the cpufreq driver (%d)\n", rc);
+ goto cleanup;
+ }
+
+ if (powernv_pstate_info.wof_enabled)
+ cpufreq_enable_boost_support();
+
+ register_reboot_notifier(&powernv_cpufreq_reboot_nb);
+ opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);
+
+ return 0;
+cleanup:
+ clean_chip_info();
+out:
+ pr_info("Platform driver disabled. System does not support PState control\n");
+ return rc;
+}
+module_init(powernv_cpufreq_init);
+
+static void __exit powernv_cpufreq_exit(void)
+{
+ cpufreq_unregister_driver(&powernv_cpufreq_driver);
+ unregister_all_notifiers();
+ clean_chip_info();
+}
+module_exit(powernv_cpufreq_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>");
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq.c b/drivers/cpufreq/ppc_cbe_cpufreq.c
new file mode 100644
index 0000000000..88afc49941
--- /dev/null
+++ b/drivers/cpufreq/ppc_cbe_cpufreq.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * cpufreq driver for the cell processor
+ *
+ * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007
+ *
+ * Author: Christian Krafft <krafft@de.ibm.com>
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include <asm/machdep.h>
+#include <asm/cell-regs.h>
+
+#include "ppc_cbe_cpufreq.h"
+
+/* the CBE supports an 8 step frequency scaling */
+static struct cpufreq_frequency_table cbe_freqs[] = {
+ {0, 1, 0},
+ {0, 2, 0},
+ {0, 3, 0},
+ {0, 4, 0},
+ {0, 5, 0},
+ {0, 6, 0},
+ {0, 8, 0},
+ {0, 10, 0},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+/*
+ * hardware specific functions
+ */
+
+static int set_pmode(unsigned int cpu, unsigned int slow_mode)
+{
+ int rc;
+
+ if (cbe_cpufreq_has_pmi)
+ rc = cbe_cpufreq_set_pmode_pmi(cpu, slow_mode);
+ else
+ rc = cbe_cpufreq_set_pmode(cpu, slow_mode);
+
+ pr_debug("register contains slow mode %d\n", cbe_cpufreq_get_pmode(cpu));
+
+ return rc;
+}
+
+/*
+ * cpufreq functions
+ */
+
+static int cbe_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *pos;
+ const u32 *max_freqp;
+ u32 max_freq;
+ int cur_pmode;
+ struct device_node *cpu;
+
+ cpu = of_get_cpu_node(policy->cpu, NULL);
+
+ if (!cpu)
+ return -ENODEV;
+
+ pr_debug("init cpufreq on CPU %d\n", policy->cpu);
+
+ /*
+ * Let's check we can actually get to the CELL regs
+ */
+ if (!cbe_get_cpu_pmd_regs(policy->cpu) ||
+ !cbe_get_cpu_mic_tm_regs(policy->cpu)) {
+ pr_info("invalid CBE regs pointers for cpufreq\n");
+ of_node_put(cpu);
+ return -EINVAL;
+ }
+
+ max_freqp = of_get_property(cpu, "clock-frequency", NULL);
+
+ of_node_put(cpu);
+
+ if (!max_freqp)
+ return -EINVAL;
+
+ /* we need the freq in kHz */
+ max_freq = *max_freqp / 1000;
+
+ pr_debug("max clock-frequency is at %u kHz\n", max_freq);
+ pr_debug("initializing frequency table\n");
+
+ /* initialize frequency table */
+ cpufreq_for_each_entry(pos, cbe_freqs) {
+ pos->frequency = max_freq / pos->driver_data;
+ pr_debug("%d: %d\n", (int)(pos - cbe_freqs), pos->frequency);
+ }
+
+ /* if DEBUG is enabled set_pmode() measures the latency
+ * of a transition */
+ policy->cpuinfo.transition_latency = 25000;
+
+ cur_pmode = cbe_cpufreq_get_pmode(policy->cpu);
+ pr_debug("current pmode is at %d\n",cur_pmode);
+
+ policy->cur = cbe_freqs[cur_pmode].frequency;
+
+#ifdef CONFIG_SMP
+ cpumask_copy(policy->cpus, cpu_sibling_mask(policy->cpu));
+#endif
+
+ policy->freq_table = cbe_freqs;
+ cbe_cpufreq_pmi_policy_init(policy);
+ return 0;
+}
+
+static int cbe_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ cbe_cpufreq_pmi_policy_exit(policy);
+ return 0;
+}
+
+static int cbe_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int cbe_pmode_new)
+{
+ pr_debug("setting frequency for cpu %d to %d kHz, " \
+ "1/%d of max frequency\n",
+ policy->cpu,
+ cbe_freqs[cbe_pmode_new].frequency,
+ cbe_freqs[cbe_pmode_new].driver_data);
+
+ return set_pmode(policy->cpu, cbe_pmode_new);
+}
+
+static struct cpufreq_driver cbe_cpufreq_driver = {
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = cbe_cpufreq_target,
+ .init = cbe_cpufreq_cpu_init,
+ .exit = cbe_cpufreq_cpu_exit,
+ .name = "cbe-cpufreq",
+ .flags = CPUFREQ_CONST_LOOPS,
+};
+
+/*
+ * module init and destoy
+ */
+
+static int __init cbe_cpufreq_init(void)
+{
+ int ret;
+
+ if (!machine_is(cell))
+ return -ENODEV;
+
+ cbe_cpufreq_pmi_init();
+
+ ret = cpufreq_register_driver(&cbe_cpufreq_driver);
+ if (ret)
+ cbe_cpufreq_pmi_exit();
+
+ return ret;
+}
+
+static void __exit cbe_cpufreq_exit(void)
+{
+ cpufreq_unregister_driver(&cbe_cpufreq_driver);
+ cbe_cpufreq_pmi_exit();
+}
+
+module_init(cbe_cpufreq_init);
+module_exit(cbe_cpufreq_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Christian Krafft <krafft@de.ibm.com>");
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq.h b/drivers/cpufreq/ppc_cbe_cpufreq.h
new file mode 100644
index 0000000000..00cd8633b0
--- /dev/null
+++ b/drivers/cpufreq/ppc_cbe_cpufreq.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ppc_cbe_cpufreq.h
+ *
+ * This file contains the definitions used by the cbe_cpufreq driver.
+ *
+ * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007
+ *
+ * Author: Christian Krafft <krafft@de.ibm.com>
+ *
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/types.h>
+
+int cbe_cpufreq_set_pmode(int cpu, unsigned int pmode);
+int cbe_cpufreq_get_pmode(int cpu);
+
+int cbe_cpufreq_set_pmode_pmi(int cpu, unsigned int pmode);
+
+#if IS_ENABLED(CONFIG_CPU_FREQ_CBE_PMI)
+extern bool cbe_cpufreq_has_pmi;
+void cbe_cpufreq_pmi_policy_init(struct cpufreq_policy *policy);
+void cbe_cpufreq_pmi_policy_exit(struct cpufreq_policy *policy);
+void cbe_cpufreq_pmi_init(void);
+void cbe_cpufreq_pmi_exit(void);
+#else
+#define cbe_cpufreq_has_pmi (0)
+static inline void cbe_cpufreq_pmi_policy_init(struct cpufreq_policy *policy) {}
+static inline void cbe_cpufreq_pmi_policy_exit(struct cpufreq_policy *policy) {}
+static inline void cbe_cpufreq_pmi_init(void) {}
+static inline void cbe_cpufreq_pmi_exit(void) {}
+#endif
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pervasive.c b/drivers/cpufreq/ppc_cbe_cpufreq_pervasive.c
new file mode 100644
index 0000000000..04830cd953
--- /dev/null
+++ b/drivers/cpufreq/ppc_cbe_cpufreq_pervasive.c
@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * pervasive backend for the cbe_cpufreq driver
+ *
+ * This driver makes use of the pervasive unit to
+ * engage the desired frequency.
+ *
+ * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007
+ *
+ * Author: Christian Krafft <krafft@de.ibm.com>
+ */
+
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/time.h>
+#include <asm/machdep.h>
+#include <asm/hw_irq.h>
+#include <asm/cell-regs.h>
+
+#include "ppc_cbe_cpufreq.h"
+
+/* to write to MIC register */
+static u64 MIC_Slow_Fast_Timer_table[] = {
+ [0 ... 7] = 0x007fc00000000000ull,
+};
+
+/* more values for the MIC */
+static u64 MIC_Slow_Next_Timer_table[] = {
+ 0x0000240000000000ull,
+ 0x0000268000000000ull,
+ 0x000029C000000000ull,
+ 0x00002D0000000000ull,
+ 0x0000300000000000ull,
+ 0x0000334000000000ull,
+ 0x000039C000000000ull,
+ 0x00003FC000000000ull,
+};
+
+
+int cbe_cpufreq_set_pmode(int cpu, unsigned int pmode)
+{
+ struct cbe_pmd_regs __iomem *pmd_regs;
+ struct cbe_mic_tm_regs __iomem *mic_tm_regs;
+ unsigned long flags;
+ u64 value;
+#ifdef DEBUG
+ long time;
+#endif
+
+ local_irq_save(flags);
+
+ mic_tm_regs = cbe_get_cpu_mic_tm_regs(cpu);
+ pmd_regs = cbe_get_cpu_pmd_regs(cpu);
+
+#ifdef DEBUG
+ time = jiffies;
+#endif
+
+ out_be64(&mic_tm_regs->slow_fast_timer_0, MIC_Slow_Fast_Timer_table[pmode]);
+ out_be64(&mic_tm_regs->slow_fast_timer_1, MIC_Slow_Fast_Timer_table[pmode]);
+
+ out_be64(&mic_tm_regs->slow_next_timer_0, MIC_Slow_Next_Timer_table[pmode]);
+ out_be64(&mic_tm_regs->slow_next_timer_1, MIC_Slow_Next_Timer_table[pmode]);
+
+ value = in_be64(&pmd_regs->pmcr);
+ /* set bits to zero */
+ value &= 0xFFFFFFFFFFFFFFF8ull;
+ /* set bits to next pmode */
+ value |= pmode;
+
+ out_be64(&pmd_regs->pmcr, value);
+
+#ifdef DEBUG
+ /* wait until new pmode appears in status register */
+ value = in_be64(&pmd_regs->pmsr) & 0x07;
+ while (value != pmode) {
+ cpu_relax();
+ value = in_be64(&pmd_regs->pmsr) & 0x07;
+ }
+
+ time = jiffies - time;
+ time = jiffies_to_msecs(time);
+ pr_debug("had to wait %lu ms for a transition using " \
+ "pervasive unit\n", time);
+#endif
+ local_irq_restore(flags);
+
+ return 0;
+}
+
+
+int cbe_cpufreq_get_pmode(int cpu)
+{
+ int ret;
+ struct cbe_pmd_regs __iomem *pmd_regs;
+
+ pmd_regs = cbe_get_cpu_pmd_regs(cpu);
+ ret = in_be64(&pmd_regs->pmsr) & 0x07;
+
+ return ret;
+}
+
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
new file mode 100644
index 0000000000..6f0c325924
--- /dev/null
+++ b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * pmi backend for the cbe_cpufreq driver
+ *
+ * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007
+ *
+ * Author: Christian Krafft <krafft@de.ibm.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/timer.h>
+#include <linux/init.h>
+#include <linux/pm_qos.h>
+#include <linux/slab.h>
+
+#include <asm/processor.h>
+#include <asm/pmi.h>
+#include <asm/cell-regs.h>
+
+#ifdef DEBUG
+#include <asm/time.h>
+#endif
+
+#include "ppc_cbe_cpufreq.h"
+
+bool cbe_cpufreq_has_pmi = false;
+EXPORT_SYMBOL_GPL(cbe_cpufreq_has_pmi);
+
+/*
+ * hardware specific functions
+ */
+
+int cbe_cpufreq_set_pmode_pmi(int cpu, unsigned int pmode)
+{
+ int ret;
+ pmi_message_t pmi_msg;
+#ifdef DEBUG
+ long time;
+#endif
+ pmi_msg.type = PMI_TYPE_FREQ_CHANGE;
+ pmi_msg.data1 = cbe_cpu_to_node(cpu);
+ pmi_msg.data2 = pmode;
+
+#ifdef DEBUG
+ time = jiffies;
+#endif
+ pmi_send_message(pmi_msg);
+
+#ifdef DEBUG
+ time = jiffies - time;
+ time = jiffies_to_msecs(time);
+ pr_debug("had to wait %lu ms for a transition using " \
+ "PMI\n", time);
+#endif
+ ret = pmi_msg.data2;
+ pr_debug("PMI returned slow mode %d\n", ret);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_set_pmode_pmi);
+
+
+static void cbe_cpufreq_handle_pmi(pmi_message_t pmi_msg)
+{
+ struct cpufreq_policy *policy;
+ struct freq_qos_request *req;
+ u8 node, slow_mode;
+ int cpu, ret;
+
+ BUG_ON(pmi_msg.type != PMI_TYPE_FREQ_CHANGE);
+
+ node = pmi_msg.data1;
+ slow_mode = pmi_msg.data2;
+
+ cpu = cbe_node_to_cpu(node);
+
+ pr_debug("cbe_handle_pmi: node: %d max_freq: %d\n", node, slow_mode);
+
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy) {
+ pr_warn("cpufreq policy not found cpu%d\n", cpu);
+ return;
+ }
+
+ req = policy->driver_data;
+
+ ret = freq_qos_update_request(req,
+ policy->freq_table[slow_mode].frequency);
+ if (ret < 0)
+ pr_warn("Failed to update freq constraint: %d\n", ret);
+ else
+ pr_debug("limiting node %d to slow mode %d\n", node, slow_mode);
+
+ cpufreq_cpu_put(policy);
+}
+
+static struct pmi_handler cbe_pmi_handler = {
+ .type = PMI_TYPE_FREQ_CHANGE,
+ .handle_pmi_message = cbe_cpufreq_handle_pmi,
+};
+
+void cbe_cpufreq_pmi_policy_init(struct cpufreq_policy *policy)
+{
+ struct freq_qos_request *req;
+ int ret;
+
+ if (!cbe_cpufreq_has_pmi)
+ return;
+
+ req = kzalloc(sizeof(*req), GFP_KERNEL);
+ if (!req)
+ return;
+
+ ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MAX,
+ policy->freq_table[0].frequency);
+ if (ret < 0) {
+ pr_err("Failed to add freq constraint (%d)\n", ret);
+ kfree(req);
+ return;
+ }
+
+ policy->driver_data = req;
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_policy_init);
+
+void cbe_cpufreq_pmi_policy_exit(struct cpufreq_policy *policy)
+{
+ struct freq_qos_request *req = policy->driver_data;
+
+ if (cbe_cpufreq_has_pmi) {
+ freq_qos_remove_request(req);
+ kfree(req);
+ }
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_policy_exit);
+
+void cbe_cpufreq_pmi_init(void)
+{
+ if (!pmi_register_handler(&cbe_pmi_handler))
+ cbe_cpufreq_has_pmi = true;
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_init);
+
+void cbe_cpufreq_pmi_exit(void)
+{
+ pmi_unregister_handler(&cbe_pmi_handler);
+ cbe_cpufreq_has_pmi = false;
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_exit);
diff --git a/drivers/cpufreq/pxa2xx-cpufreq.c b/drivers/cpufreq/pxa2xx-cpufreq.c
new file mode 100644
index 0000000000..ed1ae061a6
--- /dev/null
+++ b/drivers/cpufreq/pxa2xx-cpufreq.c
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2002,2003 Intrinsyc Software
+ *
+ * History:
+ * 31-Jul-2002 : Initial version [FB]
+ * 29-Jan-2003 : added PXA255 support [FB]
+ * 20-Apr-2003 : ported to v2.5 (Dustin McIntire, Sensoria Corp.)
+ *
+ * Note:
+ * This driver may change the memory bus clock rate, but will not do any
+ * platform specific access timing changes... for example if you have flash
+ * memory connected to CS0, you will need to register a platform specific
+ * notifier which will adjust the memory access strobes to maintain a
+ * minimum strobe width.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/regulator/consumer.h>
+#include <linux/soc/pxa/cpu.h>
+#include <linux/io.h>
+
+#ifdef DEBUG
+static unsigned int freq_debug;
+module_param(freq_debug, uint, 0);
+MODULE_PARM_DESC(freq_debug, "Set the debug messages to on=1/off=0");
+#else
+#define freq_debug 0
+#endif
+
+static struct regulator *vcc_core;
+
+static unsigned int pxa27x_maxfreq;
+module_param(pxa27x_maxfreq, uint, 0);
+MODULE_PARM_DESC(pxa27x_maxfreq, "Set the pxa27x maxfreq in MHz"
+ "(typically 624=>pxa270, 416=>pxa271, 520=>pxa272)");
+
+struct pxa_cpufreq_data {
+ struct clk *clk_core;
+};
+static struct pxa_cpufreq_data pxa_cpufreq_data;
+
+struct pxa_freqs {
+ unsigned int khz;
+ int vmin;
+ int vmax;
+};
+
+/*
+ * PXA255 definitions
+ */
+static const struct pxa_freqs pxa255_run_freqs[] =
+{
+ /* CPU MEMBUS run turbo PXbus SDRAM */
+ { 99500, -1, -1}, /* 99, 99, 50, 50 */
+ {132700, -1, -1}, /* 133, 133, 66, 66 */
+ {199100, -1, -1}, /* 199, 199, 99, 99 */
+ {265400, -1, -1}, /* 265, 265, 133, 66 */
+ {331800, -1, -1}, /* 331, 331, 166, 83 */
+ {398100, -1, -1}, /* 398, 398, 196, 99 */
+};
+
+/* Use the turbo mode frequencies for the CPUFREQ_POLICY_POWERSAVE policy */
+static const struct pxa_freqs pxa255_turbo_freqs[] =
+{
+ /* CPU run turbo PXbus SDRAM */
+ { 99500, -1, -1}, /* 99, 99, 50, 50 */
+ {199100, -1, -1}, /* 99, 199, 50, 99 */
+ {298500, -1, -1}, /* 99, 287, 50, 99 */
+ {298600, -1, -1}, /* 199, 287, 99, 99 */
+ {398100, -1, -1}, /* 199, 398, 99, 99 */
+};
+
+#define NUM_PXA25x_RUN_FREQS ARRAY_SIZE(pxa255_run_freqs)
+#define NUM_PXA25x_TURBO_FREQS ARRAY_SIZE(pxa255_turbo_freqs)
+
+static struct cpufreq_frequency_table
+ pxa255_run_freq_table[NUM_PXA25x_RUN_FREQS+1];
+static struct cpufreq_frequency_table
+ pxa255_turbo_freq_table[NUM_PXA25x_TURBO_FREQS+1];
+
+static unsigned int pxa255_turbo_table;
+module_param(pxa255_turbo_table, uint, 0);
+MODULE_PARM_DESC(pxa255_turbo_table, "Selects the frequency table (0 = run table, !0 = turbo table)");
+
+static struct pxa_freqs pxa27x_freqs[] = {
+ {104000, 900000, 1705000 },
+ {156000, 1000000, 1705000 },
+ {208000, 1180000, 1705000 },
+ {312000, 1250000, 1705000 },
+ {416000, 1350000, 1705000 },
+ {520000, 1450000, 1705000 },
+ {624000, 1550000, 1705000 }
+};
+
+#define NUM_PXA27x_FREQS ARRAY_SIZE(pxa27x_freqs)
+static struct cpufreq_frequency_table
+ pxa27x_freq_table[NUM_PXA27x_FREQS+1];
+
+#ifdef CONFIG_REGULATOR
+
+static int pxa_cpufreq_change_voltage(const struct pxa_freqs *pxa_freq)
+{
+ int ret = 0;
+ int vmin, vmax;
+
+ if (!cpu_is_pxa27x())
+ return 0;
+
+ vmin = pxa_freq->vmin;
+ vmax = pxa_freq->vmax;
+ if ((vmin == -1) || (vmax == -1))
+ return 0;
+
+ ret = regulator_set_voltage(vcc_core, vmin, vmax);
+ if (ret)
+ pr_err("Failed to set vcc_core in [%dmV..%dmV]\n", vmin, vmax);
+ return ret;
+}
+
+static void pxa_cpufreq_init_voltages(void)
+{
+ vcc_core = regulator_get(NULL, "vcc_core");
+ if (IS_ERR(vcc_core)) {
+ pr_info("Didn't find vcc_core regulator\n");
+ vcc_core = NULL;
+ } else {
+ pr_info("Found vcc_core regulator\n");
+ }
+}
+#else
+static int pxa_cpufreq_change_voltage(const struct pxa_freqs *pxa_freq)
+{
+ return 0;
+}
+
+static void pxa_cpufreq_init_voltages(void) { }
+#endif
+
+static void find_freq_tables(struct cpufreq_frequency_table **freq_table,
+ const struct pxa_freqs **pxa_freqs)
+{
+ if (cpu_is_pxa25x()) {
+ if (!pxa255_turbo_table) {
+ *pxa_freqs = pxa255_run_freqs;
+ *freq_table = pxa255_run_freq_table;
+ } else {
+ *pxa_freqs = pxa255_turbo_freqs;
+ *freq_table = pxa255_turbo_freq_table;
+ }
+ } else if (cpu_is_pxa27x()) {
+ *pxa_freqs = pxa27x_freqs;
+ *freq_table = pxa27x_freq_table;
+ } else {
+ BUG();
+ }
+}
+
+static void pxa27x_guess_max_freq(void)
+{
+ if (!pxa27x_maxfreq) {
+ pxa27x_maxfreq = 416000;
+ pr_info("PXA CPU 27x max frequency not defined (pxa27x_maxfreq), assuming pxa271 with %dkHz maxfreq\n",
+ pxa27x_maxfreq);
+ } else {
+ pxa27x_maxfreq *= 1000;
+ }
+}
+
+static unsigned int pxa_cpufreq_get(unsigned int cpu)
+{
+ struct pxa_cpufreq_data *data = cpufreq_get_driver_data();
+
+ return (unsigned int) clk_get_rate(data->clk_core) / 1000;
+}
+
+static int pxa_set_target(struct cpufreq_policy *policy, unsigned int idx)
+{
+ struct cpufreq_frequency_table *pxa_freqs_table;
+ const struct pxa_freqs *pxa_freq_settings;
+ struct pxa_cpufreq_data *data = cpufreq_get_driver_data();
+ unsigned int new_freq_cpu;
+ int ret = 0;
+
+ /* Get the current policy */
+ find_freq_tables(&pxa_freqs_table, &pxa_freq_settings);
+
+ new_freq_cpu = pxa_freq_settings[idx].khz;
+
+ if (freq_debug)
+ pr_debug("Changing CPU frequency from %d Mhz to %d Mhz\n",
+ policy->cur / 1000, new_freq_cpu / 1000);
+
+ if (vcc_core && new_freq_cpu > policy->cur) {
+ ret = pxa_cpufreq_change_voltage(&pxa_freq_settings[idx]);
+ if (ret)
+ return ret;
+ }
+
+ clk_set_rate(data->clk_core, new_freq_cpu * 1000);
+
+ /*
+ * Even if voltage setting fails, we don't report it, as the frequency
+ * change succeeded. The voltage reduction is not a critical failure,
+ * only power savings will suffer from this.
+ *
+ * Note: if the voltage change fails, and a return value is returned, a
+ * bug is triggered (seems a deadlock). Should anybody find out where,
+ * the "return 0" should become a "return ret".
+ */
+ if (vcc_core && new_freq_cpu < policy->cur)
+ ret = pxa_cpufreq_change_voltage(&pxa_freq_settings[idx]);
+
+ return 0;
+}
+
+static int pxa_cpufreq_init(struct cpufreq_policy *policy)
+{
+ int i;
+ unsigned int freq;
+ struct cpufreq_frequency_table *pxa255_freq_table;
+ const struct pxa_freqs *pxa255_freqs;
+
+ /* try to guess pxa27x cpu */
+ if (cpu_is_pxa27x())
+ pxa27x_guess_max_freq();
+
+ pxa_cpufreq_init_voltages();
+
+ /* set default policy and cpuinfo */
+ policy->cpuinfo.transition_latency = 1000; /* FIXME: 1 ms, assumed */
+
+ /* Generate pxa25x the run cpufreq_frequency_table struct */
+ for (i = 0; i < NUM_PXA25x_RUN_FREQS; i++) {
+ pxa255_run_freq_table[i].frequency = pxa255_run_freqs[i].khz;
+ pxa255_run_freq_table[i].driver_data = i;
+ }
+ pxa255_run_freq_table[i].frequency = CPUFREQ_TABLE_END;
+
+ /* Generate pxa25x the turbo cpufreq_frequency_table struct */
+ for (i = 0; i < NUM_PXA25x_TURBO_FREQS; i++) {
+ pxa255_turbo_freq_table[i].frequency =
+ pxa255_turbo_freqs[i].khz;
+ pxa255_turbo_freq_table[i].driver_data = i;
+ }
+ pxa255_turbo_freq_table[i].frequency = CPUFREQ_TABLE_END;
+
+ pxa255_turbo_table = !!pxa255_turbo_table;
+
+ /* Generate the pxa27x cpufreq_frequency_table struct */
+ for (i = 0; i < NUM_PXA27x_FREQS; i++) {
+ freq = pxa27x_freqs[i].khz;
+ if (freq > pxa27x_maxfreq)
+ break;
+ pxa27x_freq_table[i].frequency = freq;
+ pxa27x_freq_table[i].driver_data = i;
+ }
+ pxa27x_freq_table[i].driver_data = i;
+ pxa27x_freq_table[i].frequency = CPUFREQ_TABLE_END;
+
+ /*
+ * Set the policy's minimum and maximum frequencies from the tables
+ * just constructed. This sets cpuinfo.mxx_freq, min and max.
+ */
+ if (cpu_is_pxa25x()) {
+ find_freq_tables(&pxa255_freq_table, &pxa255_freqs);
+ pr_info("using %s frequency table\n",
+ pxa255_turbo_table ? "turbo" : "run");
+
+ policy->freq_table = pxa255_freq_table;
+ }
+ else if (cpu_is_pxa27x()) {
+ policy->freq_table = pxa27x_freq_table;
+ }
+
+ pr_info("frequency change support initialized\n");
+
+ return 0;
+}
+
+static struct cpufreq_driver pxa_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = pxa_set_target,
+ .init = pxa_cpufreq_init,
+ .get = pxa_cpufreq_get,
+ .name = "PXA2xx",
+ .driver_data = &pxa_cpufreq_data,
+};
+
+static int __init pxa_cpu_init(void)
+{
+ int ret = -ENODEV;
+
+ pxa_cpufreq_data.clk_core = clk_get_sys(NULL, "core");
+ if (IS_ERR(pxa_cpufreq_data.clk_core))
+ return PTR_ERR(pxa_cpufreq_data.clk_core);
+
+ if (cpu_is_pxa25x() || cpu_is_pxa27x())
+ ret = cpufreq_register_driver(&pxa_cpufreq_driver);
+ return ret;
+}
+
+static void __exit pxa_cpu_exit(void)
+{
+ cpufreq_unregister_driver(&pxa_cpufreq_driver);
+}
+
+
+MODULE_AUTHOR("Intrinsyc Software Inc.");
+MODULE_DESCRIPTION("CPU frequency changing driver for the PXA architecture");
+MODULE_LICENSE("GPL");
+module_init(pxa_cpu_init);
+module_exit(pxa_cpu_exit);
diff --git a/drivers/cpufreq/pxa3xx-cpufreq.c b/drivers/cpufreq/pxa3xx-cpufreq.c
new file mode 100644
index 0000000000..4afa48d172
--- /dev/null
+++ b/drivers/cpufreq/pxa3xx-cpufreq.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2008 Marvell International Ltd.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/soc/pxa/cpu.h>
+#include <linux/clk/pxa.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+
+#define HSS_104M (0)
+#define HSS_156M (1)
+#define HSS_208M (2)
+#define HSS_312M (3)
+
+#define SMCFS_78M (0)
+#define SMCFS_104M (2)
+#define SMCFS_208M (5)
+
+#define SFLFS_104M (0)
+#define SFLFS_156M (1)
+#define SFLFS_208M (2)
+#define SFLFS_312M (3)
+
+#define XSPCLK_156M (0)
+#define XSPCLK_NONE (3)
+
+#define DMCFS_26M (0)
+#define DMCFS_260M (3)
+
+#define ACCR_XPDIS (1 << 31) /* Core PLL Output Disable */
+#define ACCR_SPDIS (1 << 30) /* System PLL Output Disable */
+#define ACCR_D0CS (1 << 26) /* D0 Mode Clock Select */
+#define ACCR_PCCE (1 << 11) /* Power Mode Change Clock Enable */
+#define ACCR_DDR_D0CS (1 << 7) /* DDR SDRAM clock frequency in D0CS (PXA31x only) */
+
+#define ACCR_SMCFS_MASK (0x7 << 23) /* Static Memory Controller Frequency Select */
+#define ACCR_SFLFS_MASK (0x3 << 18) /* Frequency Select for Internal Memory Controller */
+#define ACCR_XSPCLK_MASK (0x3 << 16) /* Core Frequency during Frequency Change */
+#define ACCR_HSS_MASK (0x3 << 14) /* System Bus-Clock Frequency Select */
+#define ACCR_DMCFS_MASK (0x3 << 12) /* Dynamic Memory Controller Clock Frequency Select */
+#define ACCR_XN_MASK (0x7 << 8) /* Core PLL Turbo-Mode-to-Run-Mode Ratio */
+#define ACCR_XL_MASK (0x1f) /* Core PLL Run-Mode-to-Oscillator Ratio */
+
+#define ACCR_SMCFS(x) (((x) & 0x7) << 23)
+#define ACCR_SFLFS(x) (((x) & 0x3) << 18)
+#define ACCR_XSPCLK(x) (((x) & 0x3) << 16)
+#define ACCR_HSS(x) (((x) & 0x3) << 14)
+#define ACCR_DMCFS(x) (((x) & 0x3) << 12)
+#define ACCR_XN(x) (((x) & 0x7) << 8)
+#define ACCR_XL(x) ((x) & 0x1f)
+
+struct pxa3xx_freq_info {
+ unsigned int cpufreq_mhz;
+ unsigned int core_xl : 5;
+ unsigned int core_xn : 3;
+ unsigned int hss : 2;
+ unsigned int dmcfs : 2;
+ unsigned int smcfs : 3;
+ unsigned int sflfs : 2;
+ unsigned int df_clkdiv : 3;
+
+ int vcc_core; /* in mV */
+ int vcc_sram; /* in mV */
+};
+
+#define OP(cpufreq, _xl, _xn, _hss, _dmc, _smc, _sfl, _dfi, vcore, vsram) \
+{ \
+ .cpufreq_mhz = cpufreq, \
+ .core_xl = _xl, \
+ .core_xn = _xn, \
+ .hss = HSS_##_hss##M, \
+ .dmcfs = DMCFS_##_dmc##M, \
+ .smcfs = SMCFS_##_smc##M, \
+ .sflfs = SFLFS_##_sfl##M, \
+ .df_clkdiv = _dfi, \
+ .vcc_core = vcore, \
+ .vcc_sram = vsram, \
+}
+
+static struct pxa3xx_freq_info pxa300_freqs[] = {
+ /* CPU XL XN HSS DMEM SMEM SRAM DFI VCC_CORE VCC_SRAM */
+ OP(104, 8, 1, 104, 260, 78, 104, 3, 1000, 1100), /* 104MHz */
+ OP(208, 16, 1, 104, 260, 104, 156, 2, 1000, 1100), /* 208MHz */
+ OP(416, 16, 2, 156, 260, 104, 208, 2, 1100, 1200), /* 416MHz */
+ OP(624, 24, 2, 208, 260, 208, 312, 3, 1375, 1400), /* 624MHz */
+};
+
+static struct pxa3xx_freq_info pxa320_freqs[] = {
+ /* CPU XL XN HSS DMEM SMEM SRAM DFI VCC_CORE VCC_SRAM */
+ OP(104, 8, 1, 104, 260, 78, 104, 3, 1000, 1100), /* 104MHz */
+ OP(208, 16, 1, 104, 260, 104, 156, 2, 1000, 1100), /* 208MHz */
+ OP(416, 16, 2, 156, 260, 104, 208, 2, 1100, 1200), /* 416MHz */
+ OP(624, 24, 2, 208, 260, 208, 312, 3, 1375, 1400), /* 624MHz */
+ OP(806, 31, 2, 208, 260, 208, 312, 3, 1400, 1400), /* 806MHz */
+};
+
+static unsigned int pxa3xx_freqs_num;
+static struct pxa3xx_freq_info *pxa3xx_freqs;
+static struct cpufreq_frequency_table *pxa3xx_freqs_table;
+
+static int setup_freqs_table(struct cpufreq_policy *policy,
+ struct pxa3xx_freq_info *freqs, int num)
+{
+ struct cpufreq_frequency_table *table;
+ int i;
+
+ table = kcalloc(num + 1, sizeof(*table), GFP_KERNEL);
+ if (table == NULL)
+ return -ENOMEM;
+
+ for (i = 0; i < num; i++) {
+ table[i].driver_data = i;
+ table[i].frequency = freqs[i].cpufreq_mhz * 1000;
+ }
+ table[num].driver_data = i;
+ table[num].frequency = CPUFREQ_TABLE_END;
+
+ pxa3xx_freqs = freqs;
+ pxa3xx_freqs_num = num;
+ pxa3xx_freqs_table = table;
+
+ policy->freq_table = table;
+
+ return 0;
+}
+
+static void __update_core_freq(struct pxa3xx_freq_info *info)
+{
+ u32 mask, disable, enable, xclkcfg;
+
+ mask = ACCR_XN_MASK | ACCR_XL_MASK;
+ disable = mask | ACCR_XSPCLK_MASK;
+ enable = ACCR_XN(info->core_xn) | ACCR_XL(info->core_xl);
+ /* No clock until core PLL is re-locked */
+ enable |= ACCR_XSPCLK(XSPCLK_NONE);
+ xclkcfg = (info->core_xn == 2) ? 0x3 : 0x2; /* turbo bit */
+
+ pxa3xx_clk_update_accr(disable, enable, xclkcfg, mask);
+}
+
+static void __update_bus_freq(struct pxa3xx_freq_info *info)
+{
+ u32 mask, disable, enable;
+
+ mask = ACCR_SMCFS_MASK | ACCR_SFLFS_MASK | ACCR_HSS_MASK |
+ ACCR_DMCFS_MASK;
+ disable = mask;
+ enable = ACCR_SMCFS(info->smcfs) | ACCR_SFLFS(info->sflfs) |
+ ACCR_HSS(info->hss) | ACCR_DMCFS(info->dmcfs);
+
+ pxa3xx_clk_update_accr(disable, enable, 0, mask);
+}
+
+static unsigned int pxa3xx_cpufreq_get(unsigned int cpu)
+{
+ return pxa3xx_get_clk_frequency_khz(0);
+}
+
+static int pxa3xx_cpufreq_set(struct cpufreq_policy *policy, unsigned int index)
+{
+ struct pxa3xx_freq_info *next;
+ unsigned long flags;
+
+ if (policy->cpu != 0)
+ return -EINVAL;
+
+ next = &pxa3xx_freqs[index];
+
+ local_irq_save(flags);
+ __update_core_freq(next);
+ __update_bus_freq(next);
+ local_irq_restore(flags);
+
+ return 0;
+}
+
+static int pxa3xx_cpufreq_init(struct cpufreq_policy *policy)
+{
+ int ret = -EINVAL;
+
+ /* set default policy and cpuinfo */
+ policy->min = policy->cpuinfo.min_freq = 104000;
+ policy->max = policy->cpuinfo.max_freq =
+ (cpu_is_pxa320()) ? 806000 : 624000;
+ policy->cpuinfo.transition_latency = 1000; /* FIXME: 1 ms, assumed */
+
+ if (cpu_is_pxa300() || cpu_is_pxa310())
+ ret = setup_freqs_table(policy, pxa300_freqs,
+ ARRAY_SIZE(pxa300_freqs));
+
+ if (cpu_is_pxa320())
+ ret = setup_freqs_table(policy, pxa320_freqs,
+ ARRAY_SIZE(pxa320_freqs));
+
+ if (ret) {
+ pr_err("failed to setup frequency table\n");
+ return ret;
+ }
+
+ pr_info("CPUFREQ support for PXA3xx initialized\n");
+ return 0;
+}
+
+static struct cpufreq_driver pxa3xx_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = pxa3xx_cpufreq_set,
+ .init = pxa3xx_cpufreq_init,
+ .get = pxa3xx_cpufreq_get,
+ .name = "pxa3xx-cpufreq",
+};
+
+static int __init cpufreq_init(void)
+{
+ if (cpu_is_pxa3xx())
+ return cpufreq_register_driver(&pxa3xx_cpufreq_driver);
+
+ return 0;
+}
+module_init(cpufreq_init);
+
+static void __exit cpufreq_exit(void)
+{
+ cpufreq_unregister_driver(&pxa3xx_cpufreq_driver);
+}
+module_exit(cpufreq_exit);
+
+MODULE_DESCRIPTION("CPU frequency scaling driver for PXA3xx");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c
new file mode 100644
index 0000000000..70b0f21968
--- /dev/null
+++ b/drivers/cpufreq/qcom-cpufreq-hw.c
@@ -0,0 +1,760 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk-provider.h>
+#include <linux/cpufreq.h>
+#include <linux/init.h>
+#include <linux/interconnect.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/units.h>
+
+#define LUT_MAX_ENTRIES 40U
+#define LUT_SRC GENMASK(31, 30)
+#define LUT_L_VAL GENMASK(7, 0)
+#define LUT_CORE_COUNT GENMASK(18, 16)
+#define LUT_VOLT GENMASK(11, 0)
+#define CLK_HW_DIV 2
+#define LUT_TURBO_IND 1
+
+#define GT_IRQ_STATUS BIT(2)
+
+#define MAX_FREQ_DOMAINS 4
+
+struct qcom_cpufreq_soc_data {
+ u32 reg_enable;
+ u32 reg_domain_state;
+ u32 reg_dcvs_ctrl;
+ u32 reg_freq_lut;
+ u32 reg_volt_lut;
+ u32 reg_intr_clr;
+ u32 reg_current_vote;
+ u32 reg_perf_state;
+ u8 lut_row_size;
+};
+
+struct qcom_cpufreq_data {
+ void __iomem *base;
+
+ /*
+ * Mutex to synchronize between de-init sequence and re-starting LMh
+ * polling/interrupts
+ */
+ struct mutex throttle_lock;
+ int throttle_irq;
+ char irq_name[15];
+ bool cancel_throttle;
+ struct delayed_work throttle_work;
+ struct cpufreq_policy *policy;
+ struct clk_hw cpu_clk;
+
+ bool per_core_dcvs;
+};
+
+static struct {
+ struct qcom_cpufreq_data *data;
+ const struct qcom_cpufreq_soc_data *soc_data;
+} qcom_cpufreq;
+
+static unsigned long cpu_hw_rate, xo_rate;
+static bool icc_scaling_enabled;
+
+static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy,
+ unsigned long freq_khz)
+{
+ unsigned long freq_hz = freq_khz * 1000;
+ struct dev_pm_opp *opp;
+ struct device *dev;
+ int ret;
+
+ dev = get_cpu_device(policy->cpu);
+ if (!dev)
+ return -ENODEV;
+
+ opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true);
+ if (IS_ERR(opp))
+ return PTR_ERR(opp);
+
+ ret = dev_pm_opp_set_opp(dev, opp);
+ dev_pm_opp_put(opp);
+ return ret;
+}
+
+static int qcom_cpufreq_update_opp(struct device *cpu_dev,
+ unsigned long freq_khz,
+ unsigned long volt)
+{
+ unsigned long freq_hz = freq_khz * 1000;
+ int ret;
+
+ /* Skip voltage update if the opp table is not available */
+ if (!icc_scaling_enabled)
+ return dev_pm_opp_add(cpu_dev, freq_hz, volt);
+
+ ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt);
+ if (ret) {
+ dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz);
+ return ret;
+ }
+
+ return dev_pm_opp_enable(cpu_dev, freq_hz);
+}
+
+static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct qcom_cpufreq_data *data = policy->driver_data;
+ const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data;
+ unsigned long freq = policy->freq_table[index].frequency;
+ unsigned int i;
+
+ writel_relaxed(index, data->base + soc_data->reg_perf_state);
+
+ if (data->per_core_dcvs)
+ for (i = 1; i < cpumask_weight(policy->related_cpus); i++)
+ writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);
+
+ if (icc_scaling_enabled)
+ qcom_cpufreq_set_bw(policy, freq);
+
+ return 0;
+}
+
+static unsigned long qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data)
+{
+ unsigned int lval;
+
+ if (qcom_cpufreq.soc_data->reg_current_vote)
+ lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_current_vote) & 0x3ff;
+ else
+ lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_domain_state) & 0xff;
+
+ return lval * xo_rate;
+}
+
+/* Get the frequency requested by the cpufreq core for the CPU */
+static unsigned int qcom_cpufreq_get_freq(unsigned int cpu)
+{
+ struct qcom_cpufreq_data *data;
+ const struct qcom_cpufreq_soc_data *soc_data;
+ struct cpufreq_policy *policy;
+ unsigned int index;
+
+ policy = cpufreq_cpu_get_raw(cpu);
+ if (!policy)
+ return 0;
+
+ data = policy->driver_data;
+ soc_data = qcom_cpufreq.soc_data;
+
+ index = readl_relaxed(data->base + soc_data->reg_perf_state);
+ index = min(index, LUT_MAX_ENTRIES - 1);
+
+ return policy->freq_table[index].frequency;
+}
+
+static unsigned int qcom_cpufreq_hw_get(unsigned int cpu)
+{
+ struct qcom_cpufreq_data *data;
+ struct cpufreq_policy *policy;
+
+ policy = cpufreq_cpu_get_raw(cpu);
+ if (!policy)
+ return 0;
+
+ data = policy->driver_data;
+
+ if (data->throttle_irq >= 0)
+ return qcom_lmh_get_throttle_freq(data) / HZ_PER_KHZ;
+
+ return qcom_cpufreq_get_freq(cpu);
+}
+
+static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ struct qcom_cpufreq_data *data = policy->driver_data;
+ const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data;
+ unsigned int index;
+ unsigned int i;
+
+ index = policy->cached_resolved_idx;
+ writel_relaxed(index, data->base + soc_data->reg_perf_state);
+
+ if (data->per_core_dcvs)
+ for (i = 1; i < cpumask_weight(policy->related_cpus); i++)
+ writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);
+
+ return policy->freq_table[index].frequency;
+}
+
+static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
+ struct cpufreq_policy *policy)
+{
+ u32 data, src, lval, i, core_count, prev_freq = 0, freq;
+ u32 volt;
+ struct cpufreq_frequency_table *table;
+ struct dev_pm_opp *opp;
+ unsigned long rate;
+ int ret;
+ struct qcom_cpufreq_data *drv_data = policy->driver_data;
+ const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data;
+
+ table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL);
+ if (!table)
+ return -ENOMEM;
+
+ ret = dev_pm_opp_of_add_table(cpu_dev);
+ if (!ret) {
+ /* Disable all opps and cross-validate against LUT later */
+ icc_scaling_enabled = true;
+ for (rate = 0; ; rate++) {
+ opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate);
+ if (IS_ERR(opp))
+ break;
+
+ dev_pm_opp_put(opp);
+ dev_pm_opp_disable(cpu_dev, rate);
+ }
+ } else if (ret != -ENODEV) {
+ dev_err(cpu_dev, "Invalid opp table in device tree\n");
+ kfree(table);
+ return ret;
+ } else {
+ policy->fast_switch_possible = true;
+ icc_scaling_enabled = false;
+ }
+
+ for (i = 0; i < LUT_MAX_ENTRIES; i++) {
+ data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut +
+ i * soc_data->lut_row_size);
+ src = FIELD_GET(LUT_SRC, data);
+ lval = FIELD_GET(LUT_L_VAL, data);
+ core_count = FIELD_GET(LUT_CORE_COUNT, data);
+
+ data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut +
+ i * soc_data->lut_row_size);
+ volt = FIELD_GET(LUT_VOLT, data) * 1000;
+
+ if (src)
+ freq = xo_rate * lval / 1000;
+ else
+ freq = cpu_hw_rate / 1000;
+
+ if (freq != prev_freq && core_count != LUT_TURBO_IND) {
+ if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) {
+ table[i].frequency = freq;
+ dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i,
+ freq, core_count);
+ } else {
+ dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq);
+ table[i].frequency = CPUFREQ_ENTRY_INVALID;
+ }
+
+ } else if (core_count == LUT_TURBO_IND) {
+ table[i].frequency = CPUFREQ_ENTRY_INVALID;
+ }
+
+ /*
+ * Two of the same frequencies with the same core counts means
+ * end of table
+ */
+ if (i > 0 && prev_freq == freq) {
+ struct cpufreq_frequency_table *prev = &table[i - 1];
+
+ /*
+ * Only treat the last frequency that might be a boost
+ * as the boost frequency
+ */
+ if (prev->frequency == CPUFREQ_ENTRY_INVALID) {
+ if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) {
+ prev->frequency = prev_freq;
+ prev->flags = CPUFREQ_BOOST_FREQ;
+ } else {
+ dev_warn(cpu_dev, "failed to update OPP for freq=%d\n",
+ freq);
+ }
+ }
+
+ break;
+ }
+
+ prev_freq = freq;
+ }
+
+ table[i].frequency = CPUFREQ_TABLE_END;
+ policy->freq_table = table;
+ dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
+
+ return 0;
+}
+
+static void qcom_get_related_cpus(int index, struct cpumask *m)
+{
+ struct device_node *cpu_np;
+ struct of_phandle_args args;
+ int cpu, ret;
+
+ for_each_possible_cpu(cpu) {
+ cpu_np = of_cpu_device_node_get(cpu);
+ if (!cpu_np)
+ continue;
+
+ ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",
+ "#freq-domain-cells", 0,
+ &args);
+ of_node_put(cpu_np);
+ if (ret < 0)
+ continue;
+
+ if (index == args.args[0])
+ cpumask_set_cpu(cpu, m);
+ }
+}
+
+static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data)
+{
+ struct cpufreq_policy *policy = data->policy;
+ int cpu = cpumask_first(policy->related_cpus);
+ struct device *dev = get_cpu_device(cpu);
+ unsigned long freq_hz, throttled_freq;
+ struct dev_pm_opp *opp;
+
+ /*
+ * Get the h/w throttled frequency, normalize it using the
+ * registered opp table and use it to calculate thermal pressure.
+ */
+ freq_hz = qcom_lmh_get_throttle_freq(data);
+
+ opp = dev_pm_opp_find_freq_floor(dev, &freq_hz);
+ if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE)
+ opp = dev_pm_opp_find_freq_ceil(dev, &freq_hz);
+
+ if (IS_ERR(opp)) {
+ dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp);
+ } else {
+ dev_pm_opp_put(opp);
+ }
+
+ throttled_freq = freq_hz / HZ_PER_KHZ;
+
+ /* Update thermal pressure (the boost frequencies are accepted) */
+ arch_update_thermal_pressure(policy->related_cpus, throttled_freq);
+
+ /*
+ * In the unlikely case policy is unregistered do not enable
+ * polling or h/w interrupt
+ */
+ mutex_lock(&data->throttle_lock);
+ if (data->cancel_throttle)
+ goto out;
+
+ /*
+ * If h/w throttled frequency is higher than what cpufreq has requested
+ * for, then stop polling and switch back to interrupt mechanism.
+ */
+ if (throttled_freq >= qcom_cpufreq_get_freq(cpu))
+ enable_irq(data->throttle_irq);
+ else
+ mod_delayed_work(system_highpri_wq, &data->throttle_work,
+ msecs_to_jiffies(10));
+
+out:
+ mutex_unlock(&data->throttle_lock);
+}
+
+static void qcom_lmh_dcvs_poll(struct work_struct *work)
+{
+ struct qcom_cpufreq_data *data;
+
+ data = container_of(work, struct qcom_cpufreq_data, throttle_work.work);
+ qcom_lmh_dcvs_notify(data);
+}
+
+static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data)
+{
+ struct qcom_cpufreq_data *c_data = data;
+
+ /* Disable interrupt and enable polling */
+ disable_irq_nosync(c_data->throttle_irq);
+ schedule_delayed_work(&c_data->throttle_work, 0);
+
+ if (qcom_cpufreq.soc_data->reg_intr_clr)
+ writel_relaxed(GT_IRQ_STATUS,
+ c_data->base + qcom_cpufreq.soc_data->reg_intr_clr);
+
+ return IRQ_HANDLED;
+}
+
+static const struct qcom_cpufreq_soc_data qcom_soc_data = {
+ .reg_enable = 0x0,
+ .reg_dcvs_ctrl = 0xbc,
+ .reg_freq_lut = 0x110,
+ .reg_volt_lut = 0x114,
+ .reg_current_vote = 0x704,
+ .reg_perf_state = 0x920,
+ .lut_row_size = 32,
+};
+
+static const struct qcom_cpufreq_soc_data epss_soc_data = {
+ .reg_enable = 0x0,
+ .reg_domain_state = 0x20,
+ .reg_dcvs_ctrl = 0xb0,
+ .reg_freq_lut = 0x100,
+ .reg_volt_lut = 0x200,
+ .reg_intr_clr = 0x308,
+ .reg_perf_state = 0x320,
+ .lut_row_size = 4,
+};
+
+static const struct of_device_id qcom_cpufreq_hw_match[] = {
+ { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data },
+ { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data },
+ {}
+};
+MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match);
+
+static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index)
+{
+ struct qcom_cpufreq_data *data = policy->driver_data;
+ struct platform_device *pdev = cpufreq_get_driver_data();
+ int ret;
+
+ /*
+ * Look for LMh interrupt. If no interrupt line is specified /
+ * if there is an error, allow cpufreq to be enabled as usual.
+ */
+ data->throttle_irq = platform_get_irq_optional(pdev, index);
+ if (data->throttle_irq == -ENXIO)
+ return 0;
+ if (data->throttle_irq < 0)
+ return data->throttle_irq;
+
+ data->cancel_throttle = false;
+ data->policy = policy;
+
+ mutex_init(&data->throttle_lock);
+ INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll);
+
+ snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu);
+ ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq,
+ IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data);
+ if (ret) {
+ dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret);
+ return 0;
+ }
+
+ ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);
+ if (ret)
+ dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",
+ data->irq_name, data->throttle_irq);
+
+ return 0;
+}
+
+static int qcom_cpufreq_hw_cpu_online(struct cpufreq_policy *policy)
+{
+ struct qcom_cpufreq_data *data = policy->driver_data;
+ struct platform_device *pdev = cpufreq_get_driver_data();
+ int ret;
+
+ if (data->throttle_irq <= 0)
+ return 0;
+
+ mutex_lock(&data->throttle_lock);
+ data->cancel_throttle = false;
+ mutex_unlock(&data->throttle_lock);
+
+ ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);
+ if (ret)
+ dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",
+ data->irq_name, data->throttle_irq);
+
+ return ret;
+}
+
+static int qcom_cpufreq_hw_cpu_offline(struct cpufreq_policy *policy)
+{
+ struct qcom_cpufreq_data *data = policy->driver_data;
+
+ if (data->throttle_irq <= 0)
+ return 0;
+
+ mutex_lock(&data->throttle_lock);
+ data->cancel_throttle = true;
+ mutex_unlock(&data->throttle_lock);
+
+ cancel_delayed_work_sync(&data->throttle_work);
+ irq_set_affinity_and_hint(data->throttle_irq, NULL);
+ disable_irq_nosync(data->throttle_irq);
+
+ return 0;
+}
+
+static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data)
+{
+ if (data->throttle_irq <= 0)
+ return;
+
+ free_irq(data->throttle_irq, data);
+}
+
+static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy)
+{
+ struct platform_device *pdev = cpufreq_get_driver_data();
+ struct device *dev = &pdev->dev;
+ struct of_phandle_args args;
+ struct device_node *cpu_np;
+ struct device *cpu_dev;
+ struct qcom_cpufreq_data *data;
+ int ret, index;
+
+ cpu_dev = get_cpu_device(policy->cpu);
+ if (!cpu_dev) {
+ pr_err("%s: failed to get cpu%d device\n", __func__,
+ policy->cpu);
+ return -ENODEV;
+ }
+
+ cpu_np = of_cpu_device_node_get(policy->cpu);
+ if (!cpu_np)
+ return -EINVAL;
+
+ ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",
+ "#freq-domain-cells", 0, &args);
+ of_node_put(cpu_np);
+ if (ret)
+ return ret;
+
+ index = args.args[0];
+ data = &qcom_cpufreq.data[index];
+
+ /* HW should be in enabled state to proceed */
+ if (!(readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_enable) & 0x1)) {
+ dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index);
+ return -ENODEV;
+ }
+
+ if (readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_dcvs_ctrl) & 0x1)
+ data->per_core_dcvs = true;
+
+ qcom_get_related_cpus(index, policy->cpus);
+
+ policy->driver_data = data;
+ policy->dvfs_possible_from_any_cpu = true;
+
+ ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy);
+ if (ret) {
+ dev_err(dev, "Domain-%d failed to read LUT\n", index);
+ return ret;
+ }
+
+ ret = dev_pm_opp_get_opp_count(cpu_dev);
+ if (ret <= 0) {
+ dev_err(cpu_dev, "Failed to add OPPs\n");
+ return -ENODEV;
+ }
+
+ if (policy_has_boost_freq(policy)) {
+ ret = cpufreq_enable_boost_support();
+ if (ret)
+ dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
+ }
+
+ return qcom_cpufreq_hw_lmh_init(policy, index);
+}
+
+static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct device *cpu_dev = get_cpu_device(policy->cpu);
+ struct qcom_cpufreq_data *data = policy->driver_data;
+
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+ dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);
+ qcom_cpufreq_hw_lmh_exit(data);
+ kfree(policy->freq_table);
+ kfree(data);
+
+ return 0;
+}
+
+static void qcom_cpufreq_ready(struct cpufreq_policy *policy)
+{
+ struct qcom_cpufreq_data *data = policy->driver_data;
+
+ if (data->throttle_irq >= 0)
+ enable_irq(data->throttle_irq);
+}
+
+static struct freq_attr *qcom_cpufreq_hw_attr[] = {
+ &cpufreq_freq_attr_scaling_available_freqs,
+ &cpufreq_freq_attr_scaling_boost_freqs,
+ NULL
+};
+
+static struct cpufreq_driver cpufreq_qcom_hw_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = qcom_cpufreq_hw_target_index,
+ .get = qcom_cpufreq_hw_get,
+ .init = qcom_cpufreq_hw_cpu_init,
+ .exit = qcom_cpufreq_hw_cpu_exit,
+ .online = qcom_cpufreq_hw_cpu_online,
+ .offline = qcom_cpufreq_hw_cpu_offline,
+ .register_em = cpufreq_register_em_with_opp,
+ .fast_switch = qcom_cpufreq_hw_fast_switch,
+ .name = "qcom-cpufreq-hw",
+ .attr = qcom_cpufreq_hw_attr,
+ .ready = qcom_cpufreq_ready,
+};
+
+static unsigned long qcom_cpufreq_hw_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+ struct qcom_cpufreq_data *data = container_of(hw, struct qcom_cpufreq_data, cpu_clk);
+
+ return qcom_lmh_get_throttle_freq(data);
+}
+
+static const struct clk_ops qcom_cpufreq_hw_clk_ops = {
+ .recalc_rate = qcom_cpufreq_hw_recalc_rate,
+};
+
+static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev)
+{
+ struct clk_hw_onecell_data *clk_data;
+ struct device *dev = &pdev->dev;
+ struct device *cpu_dev;
+ struct clk *clk;
+ int ret, i, num_domains;
+
+ clk = clk_get(dev, "xo");
+ if (IS_ERR(clk))
+ return PTR_ERR(clk);
+
+ xo_rate = clk_get_rate(clk);
+ clk_put(clk);
+
+ clk = clk_get(dev, "alternate");
+ if (IS_ERR(clk))
+ return PTR_ERR(clk);
+
+ cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV;
+ clk_put(clk);
+
+ cpufreq_qcom_hw_driver.driver_data = pdev;
+
+ /* Check for optional interconnect paths on CPU0 */
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev)
+ return -EPROBE_DEFER;
+
+ ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL);
+ if (ret)
+ return dev_err_probe(dev, ret, "Failed to find icc paths\n");
+
+ for (num_domains = 0; num_domains < MAX_FREQ_DOMAINS; num_domains++)
+ if (!platform_get_resource(pdev, IORESOURCE_MEM, num_domains))
+ break;
+
+ qcom_cpufreq.data = devm_kzalloc(dev, sizeof(struct qcom_cpufreq_data) * num_domains,
+ GFP_KERNEL);
+ if (!qcom_cpufreq.data)
+ return -ENOMEM;
+
+ qcom_cpufreq.soc_data = of_device_get_match_data(dev);
+ if (!qcom_cpufreq.soc_data)
+ return -ENODEV;
+
+ clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, num_domains), GFP_KERNEL);
+ if (!clk_data)
+ return -ENOMEM;
+
+ clk_data->num = num_domains;
+
+ for (i = 0; i < num_domains; i++) {
+ struct qcom_cpufreq_data *data = &qcom_cpufreq.data[i];
+ struct clk_init_data clk_init = {};
+ void __iomem *base;
+
+ base = devm_platform_ioremap_resource(pdev, i);
+ if (IS_ERR(base)) {
+ dev_err(dev, "Failed to map resource index %d\n", i);
+ return PTR_ERR(base);
+ }
+
+ data->base = base;
+
+ /* Register CPU clock for each frequency domain */
+ clk_init.name = kasprintf(GFP_KERNEL, "qcom_cpufreq%d", i);
+ if (!clk_init.name)
+ return -ENOMEM;
+
+ clk_init.flags = CLK_GET_RATE_NOCACHE;
+ clk_init.ops = &qcom_cpufreq_hw_clk_ops;
+ data->cpu_clk.init = &clk_init;
+
+ ret = devm_clk_hw_register(dev, &data->cpu_clk);
+ if (ret < 0) {
+ dev_err(dev, "Failed to register clock %d: %d\n", i, ret);
+ kfree(clk_init.name);
+ return ret;
+ }
+
+ clk_data->hws[i] = &data->cpu_clk;
+ kfree(clk_init.name);
+ }
+
+ ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data);
+ if (ret < 0) {
+ dev_err(dev, "Failed to add clock provider\n");
+ return ret;
+ }
+
+ ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver);
+ if (ret)
+ dev_err(dev, "CPUFreq HW driver failed to register\n");
+ else
+ dev_dbg(dev, "QCOM CPUFreq HW driver initialized\n");
+
+ return ret;
+}
+
+static void qcom_cpufreq_hw_driver_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&cpufreq_qcom_hw_driver);
+}
+
+static struct platform_driver qcom_cpufreq_hw_driver = {
+ .probe = qcom_cpufreq_hw_driver_probe,
+ .remove_new = qcom_cpufreq_hw_driver_remove,
+ .driver = {
+ .name = "qcom-cpufreq-hw",
+ .of_match_table = qcom_cpufreq_hw_match,
+ },
+};
+
+static int __init qcom_cpufreq_hw_init(void)
+{
+ return platform_driver_register(&qcom_cpufreq_hw_driver);
+}
+postcore_initcall(qcom_cpufreq_hw_init);
+
+static void __exit qcom_cpufreq_hw_exit(void)
+{
+ platform_driver_unregister(&qcom_cpufreq_hw_driver);
+}
+module_exit(qcom_cpufreq_hw_exit);
+
+MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c
new file mode 100644
index 0000000000..84d7033e5e
--- /dev/null
+++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c
@@ -0,0 +1,413 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ */
+
+/*
+ * In Certain QCOM SoCs like apq8096 and msm8996 that have KRYO processors,
+ * the CPU frequency subset and voltage value of each OPP varies
+ * based on the silicon variant in use. Qualcomm Process Voltage Scaling Tables
+ * defines the voltage and frequency value based on the msm-id in SMEM
+ * and speedbin blown in the efuse combination.
+ * The qcom-cpufreq-nvmem driver reads the msm-id and efuse value from the SoC
+ * to provide the OPP framework with required information.
+ * This is used to determine the voltage and frequency value for each OPP of
+ * operating-points-v2 table when it is parsed by the OPP framework.
+ */
+
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/smem.h>
+
+#include <dt-bindings/arm/qcom,ids.h>
+
+struct qcom_cpufreq_drv;
+
+struct qcom_cpufreq_match_data {
+ int (*get_version)(struct device *cpu_dev,
+ struct nvmem_cell *speedbin_nvmem,
+ char **pvs_name,
+ struct qcom_cpufreq_drv *drv);
+ const char **genpd_names;
+};
+
+struct qcom_cpufreq_drv {
+ int *opp_tokens;
+ u32 versions;
+ const struct qcom_cpufreq_match_data *data;
+};
+
+static struct platform_device *cpufreq_dt_pdev, *cpufreq_pdev;
+
+static void get_krait_bin_format_a(struct device *cpu_dev,
+ int *speed, int *pvs, int *pvs_ver,
+ u8 *buf)
+{
+ u32 pte_efuse;
+
+ pte_efuse = *((u32 *)buf);
+
+ *speed = pte_efuse & 0xf;
+ if (*speed == 0xf)
+ *speed = (pte_efuse >> 4) & 0xf;
+
+ if (*speed == 0xf) {
+ *speed = 0;
+ dev_warn(cpu_dev, "Speed bin: Defaulting to %d\n", *speed);
+ } else {
+ dev_dbg(cpu_dev, "Speed bin: %d\n", *speed);
+ }
+
+ *pvs = (pte_efuse >> 10) & 0x7;
+ if (*pvs == 0x7)
+ *pvs = (pte_efuse >> 13) & 0x7;
+
+ if (*pvs == 0x7) {
+ *pvs = 0;
+ dev_warn(cpu_dev, "PVS bin: Defaulting to %d\n", *pvs);
+ } else {
+ dev_dbg(cpu_dev, "PVS bin: %d\n", *pvs);
+ }
+}
+
+static void get_krait_bin_format_b(struct device *cpu_dev,
+ int *speed, int *pvs, int *pvs_ver,
+ u8 *buf)
+{
+ u32 pte_efuse, redundant_sel;
+
+ pte_efuse = *((u32 *)buf);
+ redundant_sel = (pte_efuse >> 24) & 0x7;
+
+ *pvs_ver = (pte_efuse >> 4) & 0x3;
+
+ switch (redundant_sel) {
+ case 1:
+ *pvs = ((pte_efuse >> 28) & 0x8) | ((pte_efuse >> 6) & 0x7);
+ *speed = (pte_efuse >> 27) & 0xf;
+ break;
+ case 2:
+ *pvs = (pte_efuse >> 27) & 0xf;
+ *speed = pte_efuse & 0x7;
+ break;
+ default:
+ /* 4 bits of PVS are in efuse register bits 31, 8-6. */
+ *pvs = ((pte_efuse >> 28) & 0x8) | ((pte_efuse >> 6) & 0x7);
+ *speed = pte_efuse & 0x7;
+ }
+
+ /* Check SPEED_BIN_BLOW_STATUS */
+ if (pte_efuse & BIT(3)) {
+ dev_dbg(cpu_dev, "Speed bin: %d\n", *speed);
+ } else {
+ dev_warn(cpu_dev, "Speed bin not set. Defaulting to 0!\n");
+ *speed = 0;
+ }
+
+ /* Check PVS_BLOW_STATUS */
+ pte_efuse = *(((u32 *)buf) + 1);
+ pte_efuse &= BIT(21);
+ if (pte_efuse) {
+ dev_dbg(cpu_dev, "PVS bin: %d\n", *pvs);
+ } else {
+ dev_warn(cpu_dev, "PVS bin not set. Defaulting to 0!\n");
+ *pvs = 0;
+ }
+
+ dev_dbg(cpu_dev, "PVS version: %d\n", *pvs_ver);
+}
+
+static int qcom_cpufreq_kryo_name_version(struct device *cpu_dev,
+ struct nvmem_cell *speedbin_nvmem,
+ char **pvs_name,
+ struct qcom_cpufreq_drv *drv)
+{
+ size_t len;
+ u32 msm_id;
+ u8 *speedbin;
+ int ret;
+ *pvs_name = NULL;
+
+ ret = qcom_smem_get_soc_id(&msm_id);
+ if (ret)
+ return ret;
+
+ speedbin = nvmem_cell_read(speedbin_nvmem, &len);
+ if (IS_ERR(speedbin))
+ return PTR_ERR(speedbin);
+
+ switch (msm_id) {
+ case QCOM_ID_MSM8996:
+ case QCOM_ID_APQ8096:
+ drv->versions = 1 << (unsigned int)(*speedbin);
+ break;
+ case QCOM_ID_MSM8996SG:
+ case QCOM_ID_APQ8096SG:
+ drv->versions = 1 << ((unsigned int)(*speedbin) + 4);
+ break;
+ default:
+ BUG();
+ break;
+ }
+
+ kfree(speedbin);
+ return 0;
+}
+
+static int qcom_cpufreq_krait_name_version(struct device *cpu_dev,
+ struct nvmem_cell *speedbin_nvmem,
+ char **pvs_name,
+ struct qcom_cpufreq_drv *drv)
+{
+ int speed = 0, pvs = 0, pvs_ver = 0;
+ u8 *speedbin;
+ size_t len;
+ int ret = 0;
+
+ speedbin = nvmem_cell_read(speedbin_nvmem, &len);
+
+ if (IS_ERR(speedbin))
+ return PTR_ERR(speedbin);
+
+ switch (len) {
+ case 4:
+ get_krait_bin_format_a(cpu_dev, &speed, &pvs, &pvs_ver,
+ speedbin);
+ break;
+ case 8:
+ get_krait_bin_format_b(cpu_dev, &speed, &pvs, &pvs_ver,
+ speedbin);
+ break;
+ default:
+ dev_err(cpu_dev, "Unable to read nvmem data. Defaulting to 0!\n");
+ ret = -ENODEV;
+ goto len_error;
+ }
+
+ snprintf(*pvs_name, sizeof("speedXX-pvsXX-vXX"), "speed%d-pvs%d-v%d",
+ speed, pvs, pvs_ver);
+
+ drv->versions = (1 << speed);
+
+len_error:
+ kfree(speedbin);
+ return ret;
+}
+
+static const struct qcom_cpufreq_match_data match_data_kryo = {
+ .get_version = qcom_cpufreq_kryo_name_version,
+};
+
+static const struct qcom_cpufreq_match_data match_data_krait = {
+ .get_version = qcom_cpufreq_krait_name_version,
+};
+
+static const char *qcs404_genpd_names[] = { "cpr", NULL };
+
+static const struct qcom_cpufreq_match_data match_data_qcs404 = {
+ .genpd_names = qcs404_genpd_names,
+};
+
+static int qcom_cpufreq_probe(struct platform_device *pdev)
+{
+ struct qcom_cpufreq_drv *drv;
+ struct nvmem_cell *speedbin_nvmem;
+ struct device_node *np;
+ struct device *cpu_dev;
+ char pvs_name_buffer[] = "speedXX-pvsXX-vXX";
+ char *pvs_name = pvs_name_buffer;
+ unsigned cpu;
+ const struct of_device_id *match;
+ int ret;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev)
+ return -ENODEV;
+
+ np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
+ if (!np)
+ return -ENOENT;
+
+ ret = of_device_is_compatible(np, "operating-points-v2-kryo-cpu");
+ if (!ret) {
+ of_node_put(np);
+ return -ENOENT;
+ }
+
+ drv = kzalloc(sizeof(*drv), GFP_KERNEL);
+ if (!drv)
+ return -ENOMEM;
+
+ match = pdev->dev.platform_data;
+ drv->data = match->data;
+ if (!drv->data) {
+ ret = -ENODEV;
+ goto free_drv;
+ }
+
+ if (drv->data->get_version) {
+ speedbin_nvmem = of_nvmem_cell_get(np, NULL);
+ if (IS_ERR(speedbin_nvmem)) {
+ ret = dev_err_probe(cpu_dev, PTR_ERR(speedbin_nvmem),
+ "Could not get nvmem cell\n");
+ goto free_drv;
+ }
+
+ ret = drv->data->get_version(cpu_dev,
+ speedbin_nvmem, &pvs_name, drv);
+ if (ret) {
+ nvmem_cell_put(speedbin_nvmem);
+ goto free_drv;
+ }
+ nvmem_cell_put(speedbin_nvmem);
+ }
+ of_node_put(np);
+
+ drv->opp_tokens = kcalloc(num_possible_cpus(), sizeof(*drv->opp_tokens),
+ GFP_KERNEL);
+ if (!drv->opp_tokens) {
+ ret = -ENOMEM;
+ goto free_drv;
+ }
+
+ for_each_possible_cpu(cpu) {
+ struct dev_pm_opp_config config = {
+ .supported_hw = NULL,
+ };
+
+ cpu_dev = get_cpu_device(cpu);
+ if (NULL == cpu_dev) {
+ ret = -ENODEV;
+ goto free_opp;
+ }
+
+ if (drv->data->get_version) {
+ config.supported_hw = &drv->versions;
+ config.supported_hw_count = 1;
+
+ if (pvs_name)
+ config.prop_name = pvs_name;
+ }
+
+ if (drv->data->genpd_names) {
+ config.genpd_names = drv->data->genpd_names;
+ config.virt_devs = NULL;
+ }
+
+ if (config.supported_hw || config.genpd_names) {
+ drv->opp_tokens[cpu] = dev_pm_opp_set_config(cpu_dev, &config);
+ if (drv->opp_tokens[cpu] < 0) {
+ ret = drv->opp_tokens[cpu];
+ dev_err(cpu_dev, "Failed to set OPP config\n");
+ goto free_opp;
+ }
+ }
+ }
+
+ cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1,
+ NULL, 0);
+ if (!IS_ERR(cpufreq_dt_pdev)) {
+ platform_set_drvdata(pdev, drv);
+ return 0;
+ }
+
+ ret = PTR_ERR(cpufreq_dt_pdev);
+ dev_err(cpu_dev, "Failed to register platform device\n");
+
+free_opp:
+ for_each_possible_cpu(cpu)
+ dev_pm_opp_clear_config(drv->opp_tokens[cpu]);
+ kfree(drv->opp_tokens);
+free_drv:
+ kfree(drv);
+
+ return ret;
+}
+
+static void qcom_cpufreq_remove(struct platform_device *pdev)
+{
+ struct qcom_cpufreq_drv *drv = platform_get_drvdata(pdev);
+ unsigned int cpu;
+
+ platform_device_unregister(cpufreq_dt_pdev);
+
+ for_each_possible_cpu(cpu)
+ dev_pm_opp_clear_config(drv->opp_tokens[cpu]);
+
+ kfree(drv->opp_tokens);
+ kfree(drv);
+}
+
+static struct platform_driver qcom_cpufreq_driver = {
+ .probe = qcom_cpufreq_probe,
+ .remove_new = qcom_cpufreq_remove,
+ .driver = {
+ .name = "qcom-cpufreq-nvmem",
+ },
+};
+
+static const struct of_device_id qcom_cpufreq_match_list[] __initconst = {
+ { .compatible = "qcom,apq8096", .data = &match_data_kryo },
+ { .compatible = "qcom,msm8996", .data = &match_data_kryo },
+ { .compatible = "qcom,qcs404", .data = &match_data_qcs404 },
+ { .compatible = "qcom,ipq8064", .data = &match_data_krait },
+ { .compatible = "qcom,apq8064", .data = &match_data_krait },
+ { .compatible = "qcom,msm8974", .data = &match_data_krait },
+ { .compatible = "qcom,msm8960", .data = &match_data_krait },
+ {},
+};
+MODULE_DEVICE_TABLE(of, qcom_cpufreq_match_list);
+
+/*
+ * Since the driver depends on smem and nvmem drivers, which may
+ * return EPROBE_DEFER, all the real activity is done in the probe,
+ * which may be defered as well. The init here is only registering
+ * the driver and the platform device.
+ */
+static int __init qcom_cpufreq_init(void)
+{
+ struct device_node *np = of_find_node_by_path("/");
+ const struct of_device_id *match;
+ int ret;
+
+ if (!np)
+ return -ENODEV;
+
+ match = of_match_node(qcom_cpufreq_match_list, np);
+ of_node_put(np);
+ if (!match)
+ return -ENODEV;
+
+ ret = platform_driver_register(&qcom_cpufreq_driver);
+ if (unlikely(ret < 0))
+ return ret;
+
+ cpufreq_pdev = platform_device_register_data(NULL, "qcom-cpufreq-nvmem",
+ -1, match, sizeof(*match));
+ ret = PTR_ERR_OR_ZERO(cpufreq_pdev);
+ if (0 == ret)
+ return 0;
+
+ platform_driver_unregister(&qcom_cpufreq_driver);
+ return ret;
+}
+module_init(qcom_cpufreq_init);
+
+static void __exit qcom_cpufreq_exit(void)
+{
+ platform_device_unregister(cpufreq_pdev);
+ platform_driver_unregister(&qcom_cpufreq_driver);
+}
+module_exit(qcom_cpufreq_exit);
+
+MODULE_DESCRIPTION("Qualcomm Technologies, Inc. CPUfreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c
new file mode 100644
index 0000000000..0aecaecbb0
--- /dev/null
+++ b/drivers/cpufreq/qoriq-cpufreq.c
@@ -0,0 +1,308 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2013 Freescale Semiconductor, Inc.
+ *
+ * CPU Frequency Scaling driver for Freescale QorIQ SoCs.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/cpufreq.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+#include <linux/platform_device.h>
+
+/**
+ * struct cpu_data
+ * @pclk: the parent clock of cpu
+ * @table: frequency table
+ */
+struct cpu_data {
+ struct clk **pclk;
+ struct cpufreq_frequency_table *table;
+};
+
+/**
+ * struct soc_data - SoC specific data
+ * @flags: SOC_xxx
+ */
+struct soc_data {
+ u32 flags;
+};
+
+static u32 get_bus_freq(void)
+{
+ struct device_node *soc;
+ u32 sysfreq;
+ struct clk *pltclk;
+ int ret;
+
+ /* get platform freq by searching bus-frequency property */
+ soc = of_find_node_by_type(NULL, "soc");
+ if (soc) {
+ ret = of_property_read_u32(soc, "bus-frequency", &sysfreq);
+ of_node_put(soc);
+ if (!ret)
+ return sysfreq;
+ }
+
+ /* get platform freq by its clock name */
+ pltclk = clk_get(NULL, "cg-pll0-div1");
+ if (IS_ERR(pltclk)) {
+ pr_err("%s: can't get bus frequency %ld\n",
+ __func__, PTR_ERR(pltclk));
+ return PTR_ERR(pltclk);
+ }
+
+ return clk_get_rate(pltclk);
+}
+
+static struct clk *cpu_to_clk(int cpu)
+{
+ struct device_node *np;
+ struct clk *clk;
+
+ if (!cpu_present(cpu))
+ return NULL;
+
+ np = of_get_cpu_node(cpu, NULL);
+ if (!np)
+ return NULL;
+
+ clk = of_clk_get(np, 0);
+ of_node_put(np);
+ return clk;
+}
+
+/* traverse cpu nodes to get cpu mask of sharing clock wire */
+static void set_affected_cpus(struct cpufreq_policy *policy)
+{
+ struct cpumask *dstp = policy->cpus;
+ struct clk *clk;
+ int i;
+
+ for_each_present_cpu(i) {
+ clk = cpu_to_clk(i);
+ if (IS_ERR(clk)) {
+ pr_err("%s: no clock for cpu %d\n", __func__, i);
+ continue;
+ }
+
+ if (clk_is_match(policy->clk, clk))
+ cpumask_set_cpu(i, dstp);
+ }
+}
+
+/* reduce the duplicated frequencies in frequency table */
+static void freq_table_redup(struct cpufreq_frequency_table *freq_table,
+ int count)
+{
+ int i, j;
+
+ for (i = 1; i < count; i++) {
+ for (j = 0; j < i; j++) {
+ if (freq_table[j].frequency == CPUFREQ_ENTRY_INVALID ||
+ freq_table[j].frequency !=
+ freq_table[i].frequency)
+ continue;
+
+ freq_table[i].frequency = CPUFREQ_ENTRY_INVALID;
+ break;
+ }
+ }
+}
+
+/* sort the frequencies in frequency table in descenting order */
+static void freq_table_sort(struct cpufreq_frequency_table *freq_table,
+ int count)
+{
+ int i, j, ind;
+ unsigned int freq, max_freq;
+ struct cpufreq_frequency_table table;
+
+ for (i = 0; i < count - 1; i++) {
+ max_freq = freq_table[i].frequency;
+ ind = i;
+ for (j = i + 1; j < count; j++) {
+ freq = freq_table[j].frequency;
+ if (freq == CPUFREQ_ENTRY_INVALID ||
+ freq <= max_freq)
+ continue;
+ ind = j;
+ max_freq = freq;
+ }
+
+ if (ind != i) {
+ /* exchange the frequencies */
+ table.driver_data = freq_table[i].driver_data;
+ table.frequency = freq_table[i].frequency;
+ freq_table[i].driver_data = freq_table[ind].driver_data;
+ freq_table[i].frequency = freq_table[ind].frequency;
+ freq_table[ind].driver_data = table.driver_data;
+ freq_table[ind].frequency = table.frequency;
+ }
+ }
+}
+
+static int qoriq_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ struct device_node *np;
+ int i, count;
+ u32 freq;
+ struct clk *clk;
+ const struct clk_hw *hwclk;
+ struct cpufreq_frequency_table *table;
+ struct cpu_data *data;
+ unsigned int cpu = policy->cpu;
+ u64 u64temp;
+
+ np = of_get_cpu_node(cpu, NULL);
+ if (!np)
+ return -ENODEV;
+
+ data = kzalloc(sizeof(*data), GFP_KERNEL);
+ if (!data)
+ goto err_np;
+
+ policy->clk = of_clk_get(np, 0);
+ if (IS_ERR(policy->clk)) {
+ pr_err("%s: no clock information\n", __func__);
+ goto err_nomem2;
+ }
+
+ hwclk = __clk_get_hw(policy->clk);
+ count = clk_hw_get_num_parents(hwclk);
+
+ data->pclk = kcalloc(count, sizeof(struct clk *), GFP_KERNEL);
+ if (!data->pclk)
+ goto err_nomem2;
+
+ table = kcalloc(count + 1, sizeof(*table), GFP_KERNEL);
+ if (!table)
+ goto err_pclk;
+
+ for (i = 0; i < count; i++) {
+ clk = clk_hw_get_parent_by_index(hwclk, i)->clk;
+ data->pclk[i] = clk;
+ freq = clk_get_rate(clk);
+ table[i].frequency = freq / 1000;
+ table[i].driver_data = i;
+ }
+ freq_table_redup(table, count);
+ freq_table_sort(table, count);
+ table[i].frequency = CPUFREQ_TABLE_END;
+ policy->freq_table = table;
+ data->table = table;
+
+ /* update ->cpus if we have cluster, no harm if not */
+ set_affected_cpus(policy);
+ policy->driver_data = data;
+
+ /* Minimum transition latency is 12 platform clocks */
+ u64temp = 12ULL * NSEC_PER_SEC;
+ do_div(u64temp, get_bus_freq());
+ policy->cpuinfo.transition_latency = u64temp + 1;
+
+ of_node_put(np);
+
+ return 0;
+
+err_pclk:
+ kfree(data->pclk);
+err_nomem2:
+ kfree(data);
+err_np:
+ of_node_put(np);
+
+ return -ENODEV;
+}
+
+static int qoriq_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct cpu_data *data = policy->driver_data;
+
+ kfree(data->pclk);
+ kfree(data->table);
+ kfree(data);
+ policy->driver_data = NULL;
+
+ return 0;
+}
+
+static int qoriq_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct clk *parent;
+ struct cpu_data *data = policy->driver_data;
+
+ parent = data->pclk[data->table[index].driver_data];
+ return clk_set_parent(policy->clk, parent);
+}
+
+static struct cpufreq_driver qoriq_cpufreq_driver = {
+ .name = "qoriq_cpufreq",
+ .flags = CPUFREQ_CONST_LOOPS |
+ CPUFREQ_IS_COOLING_DEV,
+ .init = qoriq_cpufreq_cpu_init,
+ .exit = qoriq_cpufreq_cpu_exit,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = qoriq_cpufreq_target,
+ .get = cpufreq_generic_get,
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct of_device_id qoriq_cpufreq_blacklist[] = {
+ /* e6500 cannot use cpufreq due to erratum A-008083 */
+ { .compatible = "fsl,b4420-clockgen", },
+ { .compatible = "fsl,b4860-clockgen", },
+ { .compatible = "fsl,t2080-clockgen", },
+ { .compatible = "fsl,t4240-clockgen", },
+ {}
+};
+
+static int qoriq_cpufreq_probe(struct platform_device *pdev)
+{
+ int ret;
+ struct device_node *np;
+
+ np = of_find_matching_node(NULL, qoriq_cpufreq_blacklist);
+ if (np) {
+ of_node_put(np);
+ dev_info(&pdev->dev, "Disabling due to erratum A-008083");
+ return -ENODEV;
+ }
+
+ ret = cpufreq_register_driver(&qoriq_cpufreq_driver);
+ if (ret)
+ return ret;
+
+ dev_info(&pdev->dev, "Freescale QorIQ CPU frequency scaling driver\n");
+ return 0;
+}
+
+static void qoriq_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&qoriq_cpufreq_driver);
+}
+
+static struct platform_driver qoriq_cpufreq_platform_driver = {
+ .driver = {
+ .name = "qoriq-cpufreq",
+ },
+ .probe = qoriq_cpufreq_probe,
+ .remove_new = qoriq_cpufreq_remove,
+};
+module_platform_driver(qoriq_cpufreq_platform_driver);
+
+MODULE_ALIAS("platform:qoriq-cpufreq");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Tang Yuantian <Yuantian.Tang@freescale.com>");
+MODULE_DESCRIPTION("cpufreq driver for Freescale QorIQ series SoCs");
diff --git a/drivers/cpufreq/raspberrypi-cpufreq.c b/drivers/cpufreq/raspberrypi-cpufreq.c
new file mode 100644
index 0000000000..e0705cc9a5
--- /dev/null
+++ b/drivers/cpufreq/raspberrypi-cpufreq.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Raspberry Pi cpufreq driver
+ *
+ * Copyright (C) 2019, Nicolas Saenz Julienne <nsaenzjulienne@suse.de>
+ */
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+
+#define RASPBERRYPI_FREQ_INTERVAL 100000000
+
+static struct platform_device *cpufreq_dt;
+
+static int raspberrypi_cpufreq_probe(struct platform_device *pdev)
+{
+ struct device *cpu_dev;
+ unsigned long min, max;
+ unsigned long rate;
+ struct clk *clk;
+ int ret;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev) {
+ pr_err("Cannot get CPU for cpufreq driver\n");
+ return -ENODEV;
+ }
+
+ clk = clk_get(cpu_dev, NULL);
+ if (IS_ERR(clk)) {
+ dev_err(cpu_dev, "Cannot get clock for CPU0\n");
+ return PTR_ERR(clk);
+ }
+
+ /*
+ * The max and min frequencies are configurable in the Raspberry Pi
+ * firmware, so we query them at runtime.
+ */
+ min = roundup(clk_round_rate(clk, 0), RASPBERRYPI_FREQ_INTERVAL);
+ max = roundup(clk_round_rate(clk, ULONG_MAX), RASPBERRYPI_FREQ_INTERVAL);
+ clk_put(clk);
+
+ for (rate = min; rate <= max; rate += RASPBERRYPI_FREQ_INTERVAL) {
+ ret = dev_pm_opp_add(cpu_dev, rate, 0);
+ if (ret)
+ goto remove_opp;
+ }
+
+ cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+ ret = PTR_ERR_OR_ZERO(cpufreq_dt);
+ if (ret) {
+ dev_err(cpu_dev, "Failed to create platform device, %d\n", ret);
+ goto remove_opp;
+ }
+
+ return 0;
+
+remove_opp:
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+
+ return ret;
+}
+
+static void raspberrypi_cpufreq_remove(struct platform_device *pdev)
+{
+ struct device *cpu_dev;
+
+ cpu_dev = get_cpu_device(0);
+ if (cpu_dev)
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+
+ platform_device_unregister(cpufreq_dt);
+}
+
+/*
+ * Since the driver depends on clk-raspberrypi, which may return EPROBE_DEFER,
+ * all the activity is performed in the probe, which may be defered as well.
+ */
+static struct platform_driver raspberrypi_cpufreq_driver = {
+ .driver = {
+ .name = "raspberrypi-cpufreq",
+ },
+ .probe = raspberrypi_cpufreq_probe,
+ .remove_new = raspberrypi_cpufreq_remove,
+};
+module_platform_driver(raspberrypi_cpufreq_driver);
+
+MODULE_AUTHOR("Nicolas Saenz Julienne <nsaenzjulienne@suse.de");
+MODULE_DESCRIPTION("Raspberry Pi cpufreq driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:raspberrypi-cpufreq");
diff --git a/drivers/cpufreq/s3c64xx-cpufreq.c b/drivers/cpufreq/s3c64xx-cpufreq.c
new file mode 100644
index 0000000000..c6bdfc308e
--- /dev/null
+++ b/drivers/cpufreq/s3c64xx-cpufreq.c
@@ -0,0 +1,208 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2009 Wolfson Microelectronics plc
+ *
+ * S3C64xx CPUfreq Support
+ */
+
+#define pr_fmt(fmt) "cpufreq: " fmt
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/regulator/consumer.h>
+#include <linux/module.h>
+
+static struct regulator *vddarm;
+static unsigned long regulator_latency;
+
+struct s3c64xx_dvfs {
+ unsigned int vddarm_min;
+ unsigned int vddarm_max;
+};
+
+static struct s3c64xx_dvfs s3c64xx_dvfs_table[] = {
+ [0] = { 1000000, 1150000 },
+ [1] = { 1050000, 1150000 },
+ [2] = { 1100000, 1150000 },
+ [3] = { 1200000, 1350000 },
+ [4] = { 1300000, 1350000 },
+};
+
+static struct cpufreq_frequency_table s3c64xx_freq_table[] = {
+ { 0, 0, 66000 },
+ { 0, 0, 100000 },
+ { 0, 0, 133000 },
+ { 0, 1, 200000 },
+ { 0, 1, 222000 },
+ { 0, 1, 266000 },
+ { 0, 2, 333000 },
+ { 0, 2, 400000 },
+ { 0, 2, 532000 },
+ { 0, 2, 533000 },
+ { 0, 3, 667000 },
+ { 0, 4, 800000 },
+ { 0, 0, CPUFREQ_TABLE_END },
+};
+
+static int s3c64xx_cpufreq_set_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct s3c64xx_dvfs *dvfs;
+ unsigned int old_freq, new_freq;
+ int ret;
+
+ old_freq = clk_get_rate(policy->clk) / 1000;
+ new_freq = s3c64xx_freq_table[index].frequency;
+ dvfs = &s3c64xx_dvfs_table[s3c64xx_freq_table[index].driver_data];
+
+#ifdef CONFIG_REGULATOR
+ if (vddarm && new_freq > old_freq) {
+ ret = regulator_set_voltage(vddarm,
+ dvfs->vddarm_min,
+ dvfs->vddarm_max);
+ if (ret != 0) {
+ pr_err("Failed to set VDDARM for %dkHz: %d\n",
+ new_freq, ret);
+ return ret;
+ }
+ }
+#endif
+
+ ret = clk_set_rate(policy->clk, new_freq * 1000);
+ if (ret < 0) {
+ pr_err("Failed to set rate %dkHz: %d\n",
+ new_freq, ret);
+ return ret;
+ }
+
+#ifdef CONFIG_REGULATOR
+ if (vddarm && new_freq < old_freq) {
+ ret = regulator_set_voltage(vddarm,
+ dvfs->vddarm_min,
+ dvfs->vddarm_max);
+ if (ret != 0) {
+ pr_err("Failed to set VDDARM for %dkHz: %d\n",
+ new_freq, ret);
+ if (clk_set_rate(policy->clk, old_freq * 1000) < 0)
+ pr_err("Failed to restore original clock rate\n");
+
+ return ret;
+ }
+ }
+#endif
+
+ pr_debug("Set actual frequency %lukHz\n",
+ clk_get_rate(policy->clk) / 1000);
+
+ return 0;
+}
+
+#ifdef CONFIG_REGULATOR
+static void s3c64xx_cpufreq_config_regulator(void)
+{
+ int count, v, i, found;
+ struct cpufreq_frequency_table *freq;
+ struct s3c64xx_dvfs *dvfs;
+
+ count = regulator_count_voltages(vddarm);
+ if (count < 0) {
+ pr_err("Unable to check supported voltages\n");
+ }
+
+ if (!count)
+ goto out;
+
+ cpufreq_for_each_valid_entry(freq, s3c64xx_freq_table) {
+ dvfs = &s3c64xx_dvfs_table[freq->driver_data];
+ found = 0;
+
+ for (i = 0; i < count; i++) {
+ v = regulator_list_voltage(vddarm, i);
+ if (v >= dvfs->vddarm_min && v <= dvfs->vddarm_max)
+ found = 1;
+ }
+
+ if (!found) {
+ pr_debug("%dkHz unsupported by regulator\n",
+ freq->frequency);
+ freq->frequency = CPUFREQ_ENTRY_INVALID;
+ }
+ }
+
+out:
+ /* Guess based on having to do an I2C/SPI write; in future we
+ * will be able to query the regulator performance here. */
+ regulator_latency = 1 * 1000 * 1000;
+}
+#endif
+
+static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy)
+{
+ struct cpufreq_frequency_table *freq;
+
+ if (policy->cpu != 0)
+ return -EINVAL;
+
+ policy->clk = clk_get(NULL, "armclk");
+ if (IS_ERR(policy->clk)) {
+ pr_err("Unable to obtain ARMCLK: %ld\n",
+ PTR_ERR(policy->clk));
+ return PTR_ERR(policy->clk);
+ }
+
+#ifdef CONFIG_REGULATOR
+ vddarm = regulator_get(NULL, "vddarm");
+ if (IS_ERR(vddarm)) {
+ pr_err("Failed to obtain VDDARM: %ld\n", PTR_ERR(vddarm));
+ pr_err("Only frequency scaling available\n");
+ vddarm = NULL;
+ } else {
+ s3c64xx_cpufreq_config_regulator();
+ }
+#endif
+
+ cpufreq_for_each_entry(freq, s3c64xx_freq_table) {
+ unsigned long r;
+
+ /* Check for frequencies we can generate */
+ r = clk_round_rate(policy->clk, freq->frequency * 1000);
+ r /= 1000;
+ if (r != freq->frequency) {
+ pr_debug("%dkHz unsupported by clock\n",
+ freq->frequency);
+ freq->frequency = CPUFREQ_ENTRY_INVALID;
+ }
+
+ /* If we have no regulator then assume startup
+ * frequency is the maximum we can support. */
+ if (!vddarm && freq->frequency > clk_get_rate(policy->clk) / 1000)
+ freq->frequency = CPUFREQ_ENTRY_INVALID;
+ }
+
+ /* Datasheet says PLL stabalisation time (if we were to use
+ * the PLLs, which we don't currently) is ~300us worst case,
+ * but add some fudge.
+ */
+ cpufreq_generic_init(policy, s3c64xx_freq_table,
+ (500 * 1000) + regulator_latency);
+ return 0;
+}
+
+static struct cpufreq_driver s3c64xx_cpufreq_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = s3c64xx_cpufreq_set_target,
+ .get = cpufreq_generic_get,
+ .init = s3c64xx_cpufreq_driver_init,
+ .name = "s3c",
+};
+
+static int __init s3c64xx_cpufreq_init(void)
+{
+ return cpufreq_register_driver(&s3c64xx_cpufreq_driver);
+}
+module_init(s3c64xx_cpufreq_init);
diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c
new file mode 100644
index 0000000000..76c888ed8d
--- /dev/null
+++ b/drivers/cpufreq/s5pv210-cpufreq.c
@@ -0,0 +1,687 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2010 Samsung Electronics Co., Ltd.
+ * http://www.samsung.com
+ *
+ * CPU frequency scaling for S5PC110/S5PV210
+*/
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/cpufreq.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/reboot.h>
+#include <linux/regulator/consumer.h>
+
+static void __iomem *clk_base;
+static void __iomem *dmc_base[2];
+
+#define S5P_CLKREG(x) (clk_base + (x))
+
+#define S5P_APLL_LOCK S5P_CLKREG(0x00)
+#define S5P_APLL_CON S5P_CLKREG(0x100)
+#define S5P_CLK_SRC0 S5P_CLKREG(0x200)
+#define S5P_CLK_SRC2 S5P_CLKREG(0x208)
+#define S5P_CLK_DIV0 S5P_CLKREG(0x300)
+#define S5P_CLK_DIV2 S5P_CLKREG(0x308)
+#define S5P_CLK_DIV6 S5P_CLKREG(0x318)
+#define S5P_CLKDIV_STAT0 S5P_CLKREG(0x1000)
+#define S5P_CLKDIV_STAT1 S5P_CLKREG(0x1004)
+#define S5P_CLKMUX_STAT0 S5P_CLKREG(0x1100)
+#define S5P_CLKMUX_STAT1 S5P_CLKREG(0x1104)
+
+#define S5P_ARM_MCS_CON S5P_CLKREG(0x6100)
+
+/* CLKSRC0 */
+#define S5P_CLKSRC0_MUX200_SHIFT (16)
+#define S5P_CLKSRC0_MUX200_MASK (0x1 << S5P_CLKSRC0_MUX200_SHIFT)
+#define S5P_CLKSRC0_MUX166_MASK (0x1<<20)
+#define S5P_CLKSRC0_MUX133_MASK (0x1<<24)
+
+/* CLKSRC2 */
+#define S5P_CLKSRC2_G3D_SHIFT (0)
+#define S5P_CLKSRC2_G3D_MASK (0x3 << S5P_CLKSRC2_G3D_SHIFT)
+#define S5P_CLKSRC2_MFC_SHIFT (4)
+#define S5P_CLKSRC2_MFC_MASK (0x3 << S5P_CLKSRC2_MFC_SHIFT)
+
+/* CLKDIV0 */
+#define S5P_CLKDIV0_APLL_SHIFT (0)
+#define S5P_CLKDIV0_APLL_MASK (0x7 << S5P_CLKDIV0_APLL_SHIFT)
+#define S5P_CLKDIV0_A2M_SHIFT (4)
+#define S5P_CLKDIV0_A2M_MASK (0x7 << S5P_CLKDIV0_A2M_SHIFT)
+#define S5P_CLKDIV0_HCLK200_SHIFT (8)
+#define S5P_CLKDIV0_HCLK200_MASK (0x7 << S5P_CLKDIV0_HCLK200_SHIFT)
+#define S5P_CLKDIV0_PCLK100_SHIFT (12)
+#define S5P_CLKDIV0_PCLK100_MASK (0x7 << S5P_CLKDIV0_PCLK100_SHIFT)
+#define S5P_CLKDIV0_HCLK166_SHIFT (16)
+#define S5P_CLKDIV0_HCLK166_MASK (0xF << S5P_CLKDIV0_HCLK166_SHIFT)
+#define S5P_CLKDIV0_PCLK83_SHIFT (20)
+#define S5P_CLKDIV0_PCLK83_MASK (0x7 << S5P_CLKDIV0_PCLK83_SHIFT)
+#define S5P_CLKDIV0_HCLK133_SHIFT (24)
+#define S5P_CLKDIV0_HCLK133_MASK (0xF << S5P_CLKDIV0_HCLK133_SHIFT)
+#define S5P_CLKDIV0_PCLK66_SHIFT (28)
+#define S5P_CLKDIV0_PCLK66_MASK (0x7 << S5P_CLKDIV0_PCLK66_SHIFT)
+
+/* CLKDIV2 */
+#define S5P_CLKDIV2_G3D_SHIFT (0)
+#define S5P_CLKDIV2_G3D_MASK (0xF << S5P_CLKDIV2_G3D_SHIFT)
+#define S5P_CLKDIV2_MFC_SHIFT (4)
+#define S5P_CLKDIV2_MFC_MASK (0xF << S5P_CLKDIV2_MFC_SHIFT)
+
+/* CLKDIV6 */
+#define S5P_CLKDIV6_ONEDRAM_SHIFT (28)
+#define S5P_CLKDIV6_ONEDRAM_MASK (0xF << S5P_CLKDIV6_ONEDRAM_SHIFT)
+
+static struct clk *dmc0_clk;
+static struct clk *dmc1_clk;
+static DEFINE_MUTEX(set_freq_lock);
+
+/* APLL M,P,S values for 1G/800Mhz */
+#define APLL_VAL_1000 ((1 << 31) | (125 << 16) | (3 << 8) | 1)
+#define APLL_VAL_800 ((1 << 31) | (100 << 16) | (3 << 8) | 1)
+
+/* Use 800MHz when entering sleep mode */
+#define SLEEP_FREQ (800 * 1000)
+
+/* Tracks if CPU frequency can be updated anymore */
+static bool no_cpufreq_access;
+
+/*
+ * DRAM configurations to calculate refresh counter for changing
+ * frequency of memory.
+ */
+struct dram_conf {
+ unsigned long freq; /* HZ */
+ unsigned long refresh; /* DRAM refresh counter * 1000 */
+};
+
+/* DRAM configuration (DMC0 and DMC1) */
+static struct dram_conf s5pv210_dram_conf[2];
+
+enum perf_level {
+ L0, L1, L2, L3, L4,
+};
+
+enum s5pv210_mem_type {
+ LPDDR = 0x1,
+ LPDDR2 = 0x2,
+ DDR2 = 0x4,
+};
+
+enum s5pv210_dmc_port {
+ DMC0 = 0,
+ DMC1,
+};
+
+static struct cpufreq_frequency_table s5pv210_freq_table[] = {
+ {0, L0, 1000*1000},
+ {0, L1, 800*1000},
+ {0, L2, 400*1000},
+ {0, L3, 200*1000},
+ {0, L4, 100*1000},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+static struct regulator *arm_regulator;
+static struct regulator *int_regulator;
+
+struct s5pv210_dvs_conf {
+ int arm_volt; /* uV */
+ int int_volt; /* uV */
+};
+
+static const int arm_volt_max = 1350000;
+static const int int_volt_max = 1250000;
+
+static struct s5pv210_dvs_conf dvs_conf[] = {
+ [L0] = {
+ .arm_volt = 1250000,
+ .int_volt = 1100000,
+ },
+ [L1] = {
+ .arm_volt = 1200000,
+ .int_volt = 1100000,
+ },
+ [L2] = {
+ .arm_volt = 1050000,
+ .int_volt = 1100000,
+ },
+ [L3] = {
+ .arm_volt = 950000,
+ .int_volt = 1100000,
+ },
+ [L4] = {
+ .arm_volt = 950000,
+ .int_volt = 1000000,
+ },
+};
+
+static u32 clkdiv_val[5][11] = {
+ /*
+ * Clock divider value for following
+ * { APLL, A2M, HCLK_MSYS, PCLK_MSYS,
+ * HCLK_DSYS, PCLK_DSYS, HCLK_PSYS, PCLK_PSYS,
+ * ONEDRAM, MFC, G3D }
+ */
+
+ /* L0 : [1000/200/100][166/83][133/66][200/200] */
+ {0, 4, 4, 1, 3, 1, 4, 1, 3, 0, 0},
+
+ /* L1 : [800/200/100][166/83][133/66][200/200] */
+ {0, 3, 3, 1, 3, 1, 4, 1, 3, 0, 0},
+
+ /* L2 : [400/200/100][166/83][133/66][200/200] */
+ {1, 3, 1, 1, 3, 1, 4, 1, 3, 0, 0},
+
+ /* L3 : [200/200/100][166/83][133/66][200/200] */
+ {3, 3, 1, 1, 3, 1, 4, 1, 3, 0, 0},
+
+ /* L4 : [100/100/100][83/83][66/66][100/100] */
+ {7, 7, 0, 0, 7, 0, 9, 0, 7, 0, 0},
+};
+
+/*
+ * This function set DRAM refresh counter
+ * according to operating frequency of DRAM
+ * ch: DMC port number 0 or 1
+ * freq: Operating frequency of DRAM(KHz)
+ */
+static void s5pv210_set_refresh(enum s5pv210_dmc_port ch, unsigned long freq)
+{
+ unsigned long tmp, tmp1;
+ void __iomem *reg = NULL;
+
+ if (ch == DMC0) {
+ reg = (dmc_base[0] + 0x30);
+ } else if (ch == DMC1) {
+ reg = (dmc_base[1] + 0x30);
+ } else {
+ pr_err("Cannot find DMC port\n");
+ return;
+ }
+
+ /* Find current DRAM frequency */
+ tmp = s5pv210_dram_conf[ch].freq;
+
+ tmp /= freq;
+
+ tmp1 = s5pv210_dram_conf[ch].refresh;
+
+ tmp1 /= tmp;
+
+ writel_relaxed(tmp1, reg);
+}
+
+static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ unsigned long reg;
+ unsigned int priv_index;
+ unsigned int pll_changing = 0;
+ unsigned int bus_speed_changing = 0;
+ unsigned int old_freq, new_freq;
+ int arm_volt, int_volt;
+ int ret = 0;
+
+ mutex_lock(&set_freq_lock);
+
+ if (no_cpufreq_access) {
+ pr_err("Denied access to %s as it is disabled temporarily\n",
+ __func__);
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ old_freq = policy->cur;
+ new_freq = s5pv210_freq_table[index].frequency;
+
+ /* Finding current running level index */
+ priv_index = cpufreq_table_find_index_h(policy, old_freq, false);
+
+ arm_volt = dvs_conf[index].arm_volt;
+ int_volt = dvs_conf[index].int_volt;
+
+ if (new_freq > old_freq) {
+ ret = regulator_set_voltage(arm_regulator,
+ arm_volt, arm_volt_max);
+ if (ret)
+ goto exit;
+
+ ret = regulator_set_voltage(int_regulator,
+ int_volt, int_volt_max);
+ if (ret)
+ goto exit;
+ }
+
+ /* Check if there need to change PLL */
+ if ((index == L0) || (priv_index == L0))
+ pll_changing = 1;
+
+ /* Check if there need to change System bus clock */
+ if ((index == L4) || (priv_index == L4))
+ bus_speed_changing = 1;
+
+ if (bus_speed_changing) {
+ /*
+ * Reconfigure DRAM refresh counter value for minimum
+ * temporary clock while changing divider.
+ * expected clock is 83Mhz : 7.8usec/(1/83Mhz) = 0x287
+ */
+ if (pll_changing)
+ s5pv210_set_refresh(DMC1, 83000);
+ else
+ s5pv210_set_refresh(DMC1, 100000);
+
+ s5pv210_set_refresh(DMC0, 83000);
+ }
+
+ /*
+ * APLL should be changed in this level
+ * APLL -> MPLL(for stable transition) -> APLL
+ * Some clock source's clock API are not prepared.
+ * Do not use clock API in below code.
+ */
+ if (pll_changing) {
+ /*
+ * 1. Temporary Change divider for MFC and G3D
+ * SCLKA2M(200/1=200)->(200/4=50)Mhz
+ */
+ reg = readl_relaxed(S5P_CLK_DIV2);
+ reg &= ~(S5P_CLKDIV2_G3D_MASK | S5P_CLKDIV2_MFC_MASK);
+ reg |= (3 << S5P_CLKDIV2_G3D_SHIFT) |
+ (3 << S5P_CLKDIV2_MFC_SHIFT);
+ writel_relaxed(reg, S5P_CLK_DIV2);
+
+ /* For MFC, G3D dividing */
+ do {
+ reg = readl_relaxed(S5P_CLKDIV_STAT0);
+ } while (reg & ((1 << 16) | (1 << 17)));
+
+ /*
+ * 2. Change SCLKA2M(200Mhz)to SCLKMPLL in MFC_MUX, G3D MUX
+ * (200/4=50)->(667/4=166)Mhz
+ */
+ reg = readl_relaxed(S5P_CLK_SRC2);
+ reg &= ~(S5P_CLKSRC2_G3D_MASK | S5P_CLKSRC2_MFC_MASK);
+ reg |= (1 << S5P_CLKSRC2_G3D_SHIFT) |
+ (1 << S5P_CLKSRC2_MFC_SHIFT);
+ writel_relaxed(reg, S5P_CLK_SRC2);
+
+ do {
+ reg = readl_relaxed(S5P_CLKMUX_STAT1);
+ } while (reg & ((1 << 7) | (1 << 3)));
+
+ /*
+ * 3. DMC1 refresh count for 133Mhz if (index == L4) is
+ * true refresh counter is already programmed in upper
+ * code. 0x287@83Mhz
+ */
+ if (!bus_speed_changing)
+ s5pv210_set_refresh(DMC1, 133000);
+
+ /* 4. SCLKAPLL -> SCLKMPLL */
+ reg = readl_relaxed(S5P_CLK_SRC0);
+ reg &= ~(S5P_CLKSRC0_MUX200_MASK);
+ reg |= (0x1 << S5P_CLKSRC0_MUX200_SHIFT);
+ writel_relaxed(reg, S5P_CLK_SRC0);
+
+ do {
+ reg = readl_relaxed(S5P_CLKMUX_STAT0);
+ } while (reg & (0x1 << 18));
+
+ }
+
+ /* Change divider */
+ reg = readl_relaxed(S5P_CLK_DIV0);
+
+ reg &= ~(S5P_CLKDIV0_APLL_MASK | S5P_CLKDIV0_A2M_MASK |
+ S5P_CLKDIV0_HCLK200_MASK | S5P_CLKDIV0_PCLK100_MASK |
+ S5P_CLKDIV0_HCLK166_MASK | S5P_CLKDIV0_PCLK83_MASK |
+ S5P_CLKDIV0_HCLK133_MASK | S5P_CLKDIV0_PCLK66_MASK);
+
+ reg |= ((clkdiv_val[index][0] << S5P_CLKDIV0_APLL_SHIFT) |
+ (clkdiv_val[index][1] << S5P_CLKDIV0_A2M_SHIFT) |
+ (clkdiv_val[index][2] << S5P_CLKDIV0_HCLK200_SHIFT) |
+ (clkdiv_val[index][3] << S5P_CLKDIV0_PCLK100_SHIFT) |
+ (clkdiv_val[index][4] << S5P_CLKDIV0_HCLK166_SHIFT) |
+ (clkdiv_val[index][5] << S5P_CLKDIV0_PCLK83_SHIFT) |
+ (clkdiv_val[index][6] << S5P_CLKDIV0_HCLK133_SHIFT) |
+ (clkdiv_val[index][7] << S5P_CLKDIV0_PCLK66_SHIFT));
+
+ writel_relaxed(reg, S5P_CLK_DIV0);
+
+ do {
+ reg = readl_relaxed(S5P_CLKDIV_STAT0);
+ } while (reg & 0xff);
+
+ /* ARM MCS value changed */
+ reg = readl_relaxed(S5P_ARM_MCS_CON);
+ reg &= ~0x3;
+ if (index >= L3)
+ reg |= 0x3;
+ else
+ reg |= 0x1;
+
+ writel_relaxed(reg, S5P_ARM_MCS_CON);
+
+ if (pll_changing) {
+ /* 5. Set Lock time = 30us*24Mhz = 0x2cf */
+ writel_relaxed(0x2cf, S5P_APLL_LOCK);
+
+ /*
+ * 6. Turn on APLL
+ * 6-1. Set PMS values
+ * 6-2. Wait until the PLL is locked
+ */
+ if (index == L0)
+ writel_relaxed(APLL_VAL_1000, S5P_APLL_CON);
+ else
+ writel_relaxed(APLL_VAL_800, S5P_APLL_CON);
+
+ do {
+ reg = readl_relaxed(S5P_APLL_CON);
+ } while (!(reg & (0x1 << 29)));
+
+ /*
+ * 7. Change source clock from SCLKMPLL(667Mhz)
+ * to SCLKA2M(200Mhz) in MFC_MUX and G3D MUX
+ * (667/4=166)->(200/4=50)Mhz
+ */
+ reg = readl_relaxed(S5P_CLK_SRC2);
+ reg &= ~(S5P_CLKSRC2_G3D_MASK | S5P_CLKSRC2_MFC_MASK);
+ reg |= (0 << S5P_CLKSRC2_G3D_SHIFT) |
+ (0 << S5P_CLKSRC2_MFC_SHIFT);
+ writel_relaxed(reg, S5P_CLK_SRC2);
+
+ do {
+ reg = readl_relaxed(S5P_CLKMUX_STAT1);
+ } while (reg & ((1 << 7) | (1 << 3)));
+
+ /*
+ * 8. Change divider for MFC and G3D
+ * (200/4=50)->(200/1=200)Mhz
+ */
+ reg = readl_relaxed(S5P_CLK_DIV2);
+ reg &= ~(S5P_CLKDIV2_G3D_MASK | S5P_CLKDIV2_MFC_MASK);
+ reg |= (clkdiv_val[index][10] << S5P_CLKDIV2_G3D_SHIFT) |
+ (clkdiv_val[index][9] << S5P_CLKDIV2_MFC_SHIFT);
+ writel_relaxed(reg, S5P_CLK_DIV2);
+
+ /* For MFC, G3D dividing */
+ do {
+ reg = readl_relaxed(S5P_CLKDIV_STAT0);
+ } while (reg & ((1 << 16) | (1 << 17)));
+
+ /* 9. Change MPLL to APLL in MSYS_MUX */
+ reg = readl_relaxed(S5P_CLK_SRC0);
+ reg &= ~(S5P_CLKSRC0_MUX200_MASK);
+ reg |= (0x0 << S5P_CLKSRC0_MUX200_SHIFT);
+ writel_relaxed(reg, S5P_CLK_SRC0);
+
+ do {
+ reg = readl_relaxed(S5P_CLKMUX_STAT0);
+ } while (reg & (0x1 << 18));
+
+ /*
+ * 10. DMC1 refresh counter
+ * L4 : DMC1 = 100Mhz 7.8us/(1/100) = 0x30c
+ * Others : DMC1 = 200Mhz 7.8us/(1/200) = 0x618
+ */
+ if (!bus_speed_changing)
+ s5pv210_set_refresh(DMC1, 200000);
+ }
+
+ /*
+ * L4 level needs to change memory bus speed, hence ONEDRAM clock
+ * divider and memory refresh parameter should be changed
+ */
+ if (bus_speed_changing) {
+ reg = readl_relaxed(S5P_CLK_DIV6);
+ reg &= ~S5P_CLKDIV6_ONEDRAM_MASK;
+ reg |= (clkdiv_val[index][8] << S5P_CLKDIV6_ONEDRAM_SHIFT);
+ writel_relaxed(reg, S5P_CLK_DIV6);
+
+ do {
+ reg = readl_relaxed(S5P_CLKDIV_STAT1);
+ } while (reg & (1 << 15));
+
+ /* Reconfigure DRAM refresh counter value */
+ if (index != L4) {
+ /*
+ * DMC0 : 166Mhz
+ * DMC1 : 200Mhz
+ */
+ s5pv210_set_refresh(DMC0, 166000);
+ s5pv210_set_refresh(DMC1, 200000);
+ } else {
+ /*
+ * DMC0 : 83Mhz
+ * DMC1 : 100Mhz
+ */
+ s5pv210_set_refresh(DMC0, 83000);
+ s5pv210_set_refresh(DMC1, 100000);
+ }
+ }
+
+ if (new_freq < old_freq) {
+ regulator_set_voltage(int_regulator,
+ int_volt, int_volt_max);
+
+ regulator_set_voltage(arm_regulator,
+ arm_volt, arm_volt_max);
+ }
+
+ pr_debug("Perf changed[L%d]\n", index);
+
+exit:
+ mutex_unlock(&set_freq_lock);
+ return ret;
+}
+
+static int check_mem_type(void __iomem *dmc_reg)
+{
+ unsigned long val;
+
+ val = readl_relaxed(dmc_reg + 0x4);
+ val = (val & (0xf << 8));
+
+ return val >> 8;
+}
+
+static int s5pv210_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned long mem_type;
+ int ret;
+
+ policy->clk = clk_get(NULL, "armclk");
+ if (IS_ERR(policy->clk))
+ return PTR_ERR(policy->clk);
+
+ dmc0_clk = clk_get(NULL, "sclk_dmc0");
+ if (IS_ERR(dmc0_clk)) {
+ ret = PTR_ERR(dmc0_clk);
+ goto out_dmc0;
+ }
+
+ dmc1_clk = clk_get(NULL, "hclk_msys");
+ if (IS_ERR(dmc1_clk)) {
+ ret = PTR_ERR(dmc1_clk);
+ goto out_dmc1;
+ }
+
+ if (policy->cpu != 0) {
+ ret = -EINVAL;
+ goto out_dmc1;
+ }
+
+ /*
+ * check_mem_type : This driver only support LPDDR & LPDDR2.
+ * other memory type is not supported.
+ */
+ mem_type = check_mem_type(dmc_base[0]);
+
+ if ((mem_type != LPDDR) && (mem_type != LPDDR2)) {
+ pr_err("CPUFreq doesn't support this memory type\n");
+ ret = -EINVAL;
+ goto out_dmc1;
+ }
+
+ /* Find current refresh counter and frequency each DMC */
+ s5pv210_dram_conf[0].refresh = (readl_relaxed(dmc_base[0] + 0x30) * 1000);
+ s5pv210_dram_conf[0].freq = clk_get_rate(dmc0_clk);
+
+ s5pv210_dram_conf[1].refresh = (readl_relaxed(dmc_base[1] + 0x30) * 1000);
+ s5pv210_dram_conf[1].freq = clk_get_rate(dmc1_clk);
+
+ policy->suspend_freq = SLEEP_FREQ;
+ cpufreq_generic_init(policy, s5pv210_freq_table, 40000);
+ return 0;
+
+out_dmc1:
+ clk_put(dmc0_clk);
+out_dmc0:
+ clk_put(policy->clk);
+ return ret;
+}
+
+static int s5pv210_cpufreq_reboot_notifier_event(struct notifier_block *this,
+ unsigned long event, void *ptr)
+{
+ int ret;
+ struct cpufreq_policy *policy;
+
+ policy = cpufreq_cpu_get(0);
+ if (!policy) {
+ pr_debug("cpufreq: get no policy for cpu0\n");
+ return NOTIFY_BAD;
+ }
+
+ ret = cpufreq_driver_target(policy, SLEEP_FREQ, 0);
+ cpufreq_cpu_put(policy);
+
+ if (ret < 0)
+ return NOTIFY_BAD;
+
+ no_cpufreq_access = true;
+ return NOTIFY_DONE;
+}
+
+static struct cpufreq_driver s5pv210_driver = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = s5pv210_target,
+ .get = cpufreq_generic_get,
+ .init = s5pv210_cpu_init,
+ .name = "s5pv210",
+ .suspend = cpufreq_generic_suspend,
+ .resume = cpufreq_generic_suspend, /* We need to set SLEEP FREQ again */
+};
+
+static struct notifier_block s5pv210_cpufreq_reboot_notifier = {
+ .notifier_call = s5pv210_cpufreq_reboot_notifier_event,
+};
+
+static int s5pv210_cpufreq_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct device_node *np;
+ int id, result = 0;
+
+ /*
+ * HACK: This is a temporary workaround to get access to clock
+ * and DMC controller registers directly and remove static mappings
+ * and dependencies on platform headers. It is necessary to enable
+ * S5PV210 multi-platform support and will be removed together with
+ * this whole driver as soon as S5PV210 gets migrated to use
+ * cpufreq-dt driver.
+ */
+ arm_regulator = regulator_get(NULL, "vddarm");
+ if (IS_ERR(arm_regulator))
+ return dev_err_probe(dev, PTR_ERR(arm_regulator),
+ "failed to get regulator vddarm\n");
+
+ int_regulator = regulator_get(NULL, "vddint");
+ if (IS_ERR(int_regulator)) {
+ result = dev_err_probe(dev, PTR_ERR(int_regulator),
+ "failed to get regulator vddint\n");
+ goto err_int_regulator;
+ }
+
+ np = of_find_compatible_node(NULL, NULL, "samsung,s5pv210-clock");
+ if (!np) {
+ dev_err(dev, "failed to find clock controller DT node\n");
+ result = -ENODEV;
+ goto err_clock;
+ }
+
+ clk_base = of_iomap(np, 0);
+ of_node_put(np);
+ if (!clk_base) {
+ dev_err(dev, "failed to map clock registers\n");
+ result = -EFAULT;
+ goto err_clock;
+ }
+
+ for_each_compatible_node(np, NULL, "samsung,s5pv210-dmc") {
+ id = of_alias_get_id(np, "dmc");
+ if (id < 0 || id >= ARRAY_SIZE(dmc_base)) {
+ dev_err(dev, "failed to get alias of dmc node '%pOFn'\n", np);
+ of_node_put(np);
+ result = id;
+ goto err_clk_base;
+ }
+
+ dmc_base[id] = of_iomap(np, 0);
+ if (!dmc_base[id]) {
+ dev_err(dev, "failed to map dmc%d registers\n", id);
+ of_node_put(np);
+ result = -EFAULT;
+ goto err_dmc;
+ }
+ }
+
+ for (id = 0; id < ARRAY_SIZE(dmc_base); ++id) {
+ if (!dmc_base[id]) {
+ dev_err(dev, "failed to find dmc%d node\n", id);
+ result = -ENODEV;
+ goto err_dmc;
+ }
+ }
+
+ register_reboot_notifier(&s5pv210_cpufreq_reboot_notifier);
+
+ return cpufreq_register_driver(&s5pv210_driver);
+
+err_dmc:
+ for (id = 0; id < ARRAY_SIZE(dmc_base); ++id)
+ if (dmc_base[id]) {
+ iounmap(dmc_base[id]);
+ dmc_base[id] = NULL;
+ }
+
+err_clk_base:
+ iounmap(clk_base);
+
+err_clock:
+ regulator_put(int_regulator);
+
+err_int_regulator:
+ regulator_put(arm_regulator);
+
+ return result;
+}
+
+static struct platform_driver s5pv210_cpufreq_platdrv = {
+ .driver = {
+ .name = "s5pv210-cpufreq",
+ },
+ .probe = s5pv210_cpufreq_probe,
+};
+builtin_platform_driver(s5pv210_cpufreq_platdrv);
diff --git a/drivers/cpufreq/sa1110-cpufreq.c b/drivers/cpufreq/sa1110-cpufreq.c
new file mode 100644
index 0000000000..bb7f591a8b
--- /dev/null
+++ b/drivers/cpufreq/sa1110-cpufreq.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/arch/arm/mach-sa1100/cpu-sa1110.c
+ *
+ * Copyright (C) 2001 Russell King
+ *
+ * Note: there are two erratas that apply to the SA1110 here:
+ * 7 - SDRAM auto-power-up failure (rev A0)
+ * 13 - Corruption of internal register reads/writes following
+ * SDRAM reads (rev A0, B0, B1)
+ *
+ * We ignore rev. A0 and B0 devices; I don't think they're worth supporting.
+ *
+ * The SDRAM type can be passed on the command line as cpu_sa1110.sdram=type
+ */
+#include <linux/cpufreq.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+
+#include <asm/cputype.h>
+#include <asm/mach-types.h>
+
+#include <mach/generic.h>
+#include <mach/hardware.h>
+
+#undef DEBUG
+
+struct sdram_params {
+ const char name[20];
+ u_char rows; /* bits */
+ u_char cas_latency; /* cycles */
+ u_char tck; /* clock cycle time (ns) */
+ u_char trcd; /* activate to r/w (ns) */
+ u_char trp; /* precharge to activate (ns) */
+ u_char twr; /* write recovery time (ns) */
+ u_short refresh; /* refresh time for array (us) */
+};
+
+struct sdram_info {
+ u_int mdcnfg;
+ u_int mdrefr;
+ u_int mdcas[3];
+};
+
+static struct sdram_params sdram_tbl[] __initdata = {
+ { /* Toshiba TC59SM716 CL2 */
+ .name = "TC59SM716-CL2",
+ .rows = 12,
+ .tck = 10,
+ .trcd = 20,
+ .trp = 20,
+ .twr = 10,
+ .refresh = 64000,
+ .cas_latency = 2,
+ }, { /* Toshiba TC59SM716 CL3 */
+ .name = "TC59SM716-CL3",
+ .rows = 12,
+ .tck = 8,
+ .trcd = 20,
+ .trp = 20,
+ .twr = 8,
+ .refresh = 64000,
+ .cas_latency = 3,
+ }, { /* Samsung K4S641632D TC75 */
+ .name = "K4S641632D",
+ .rows = 14,
+ .tck = 9,
+ .trcd = 27,
+ .trp = 20,
+ .twr = 9,
+ .refresh = 64000,
+ .cas_latency = 3,
+ }, { /* Samsung K4S281632B-1H */
+ .name = "K4S281632B-1H",
+ .rows = 12,
+ .tck = 10,
+ .trp = 20,
+ .twr = 10,
+ .refresh = 64000,
+ .cas_latency = 3,
+ }, { /* Samsung KM416S4030CT */
+ .name = "KM416S4030CT",
+ .rows = 13,
+ .tck = 8,
+ .trcd = 24, /* 3 CLKs */
+ .trp = 24, /* 3 CLKs */
+ .twr = 16, /* Trdl: 2 CLKs */
+ .refresh = 64000,
+ .cas_latency = 3,
+ }, { /* Winbond W982516AH75L CL3 */
+ .name = "W982516AH75L",
+ .rows = 16,
+ .tck = 8,
+ .trcd = 20,
+ .trp = 20,
+ .twr = 8,
+ .refresh = 64000,
+ .cas_latency = 3,
+ }, { /* Micron MT48LC8M16A2TG-75 */
+ .name = "MT48LC8M16A2TG-75",
+ .rows = 12,
+ .tck = 8,
+ .trcd = 20,
+ .trp = 20,
+ .twr = 8,
+ .refresh = 64000,
+ .cas_latency = 3,
+ },
+};
+
+static struct sdram_params sdram_params;
+
+/*
+ * Given a period in ns and frequency in khz, calculate the number of
+ * cycles of frequency in period. Note that we round up to the next
+ * cycle, even if we are only slightly over.
+ */
+static inline u_int ns_to_cycles(u_int ns, u_int khz)
+{
+ return (ns * khz + 999999) / 1000000;
+}
+
+/*
+ * Create the MDCAS register bit pattern.
+ */
+static inline void set_mdcas(u_int *mdcas, int delayed, u_int rcd)
+{
+ u_int shift;
+
+ rcd = 2 * rcd - 1;
+ shift = delayed + 1 + rcd;
+
+ mdcas[0] = (1 << rcd) - 1;
+ mdcas[0] |= 0x55555555 << shift;
+ mdcas[1] = mdcas[2] = 0x55555555 << (shift & 1);
+}
+
+static void
+sdram_calculate_timing(struct sdram_info *sd, u_int cpu_khz,
+ struct sdram_params *sdram)
+{
+ u_int mem_khz, sd_khz, trp, twr;
+
+ mem_khz = cpu_khz / 2;
+ sd_khz = mem_khz;
+
+ /*
+ * If SDCLK would invalidate the SDRAM timings,
+ * run SDCLK at half speed.
+ *
+ * CPU steppings prior to B2 must either run the memory at
+ * half speed or use delayed read latching (errata 13).
+ */
+ if ((ns_to_cycles(sdram->tck, sd_khz) > 1) ||
+ (read_cpuid_revision() < ARM_CPU_REV_SA1110_B2 && sd_khz < 62000))
+ sd_khz /= 2;
+
+ sd->mdcnfg = MDCNFG & 0x007f007f;
+
+ twr = ns_to_cycles(sdram->twr, mem_khz);
+
+ /* trp should always be >1 */
+ trp = ns_to_cycles(sdram->trp, mem_khz) - 1;
+ if (trp < 1)
+ trp = 1;
+
+ sd->mdcnfg |= trp << 8;
+ sd->mdcnfg |= trp << 24;
+ sd->mdcnfg |= sdram->cas_latency << 12;
+ sd->mdcnfg |= sdram->cas_latency << 28;
+ sd->mdcnfg |= twr << 14;
+ sd->mdcnfg |= twr << 30;
+
+ sd->mdrefr = MDREFR & 0xffbffff0;
+ sd->mdrefr |= 7;
+
+ if (sd_khz != mem_khz)
+ sd->mdrefr |= MDREFR_K1DB2;
+
+ /* initial number of '1's in MDCAS + 1 */
+ set_mdcas(sd->mdcas, sd_khz >= 62000,
+ ns_to_cycles(sdram->trcd, mem_khz));
+
+#ifdef DEBUG
+ printk(KERN_DEBUG "MDCNFG: %08x MDREFR: %08x MDCAS0: %08x MDCAS1: %08x MDCAS2: %08x\n",
+ sd->mdcnfg, sd->mdrefr, sd->mdcas[0], sd->mdcas[1],
+ sd->mdcas[2]);
+#endif
+}
+
+/*
+ * Set the SDRAM refresh rate.
+ */
+static inline void sdram_set_refresh(u_int dri)
+{
+ MDREFR = (MDREFR & 0xffff000f) | (dri << 4);
+ (void) MDREFR;
+}
+
+/*
+ * Update the refresh period. We do this such that we always refresh
+ * the SDRAMs within their permissible period. The refresh period is
+ * always a multiple of the memory clock (fixed at cpu_clock / 2).
+ *
+ * FIXME: we don't currently take account of burst accesses here,
+ * but neither do Intels DM nor Angel.
+ */
+static void
+sdram_update_refresh(u_int cpu_khz, struct sdram_params *sdram)
+{
+ u_int ns_row = (sdram->refresh * 1000) >> sdram->rows;
+ u_int dri = ns_to_cycles(ns_row, cpu_khz / 2) / 32;
+
+#ifdef DEBUG
+ mdelay(250);
+ printk(KERN_DEBUG "new dri value = %d\n", dri);
+#endif
+
+ sdram_set_refresh(dri);
+}
+
+/*
+ * Ok, set the CPU frequency.
+ */
+static int sa1110_target(struct cpufreq_policy *policy, unsigned int ppcr)
+{
+ struct sdram_params *sdram = &sdram_params;
+ struct sdram_info sd;
+ unsigned long flags;
+ unsigned int unused;
+
+ sdram_calculate_timing(&sd, sa11x0_freq_table[ppcr].frequency, sdram);
+
+#if 0
+ /*
+ * These values are wrong according to the SA1110 documentation
+ * and errata, but they seem to work. Need to get a storage
+ * scope on to the SDRAM signals to work out why.
+ */
+ if (policy->max < 147500) {
+ sd.mdrefr |= MDREFR_K1DB2;
+ sd.mdcas[0] = 0xaaaaaa7f;
+ } else {
+ sd.mdrefr &= ~MDREFR_K1DB2;
+ sd.mdcas[0] = 0xaaaaaa9f;
+ }
+ sd.mdcas[1] = 0xaaaaaaaa;
+ sd.mdcas[2] = 0xaaaaaaaa;
+#endif
+
+ /*
+ * The clock could be going away for some time. Set the SDRAMs
+ * to refresh rapidly (every 64 memory clock cycles). To get
+ * through the whole array, we need to wait 262144 mclk cycles.
+ * We wait 20ms to be safe.
+ */
+ sdram_set_refresh(2);
+ if (!irqs_disabled())
+ msleep(20);
+ else
+ mdelay(20);
+
+ /*
+ * Reprogram the DRAM timings with interrupts disabled, and
+ * ensure that we are doing this within a complete cache line.
+ * This means that we won't access SDRAM for the duration of
+ * the programming.
+ */
+ local_irq_save(flags);
+ asm("mcr p15, 0, %0, c7, c10, 4" : : "r" (0));
+ udelay(10);
+ __asm__ __volatile__("\n\
+ b 2f \n\
+ .align 5 \n\
+1: str %3, [%1, #0] @ MDCNFG \n\
+ str %4, [%1, #28] @ MDREFR \n\
+ str %5, [%1, #4] @ MDCAS0 \n\
+ str %6, [%1, #8] @ MDCAS1 \n\
+ str %7, [%1, #12] @ MDCAS2 \n\
+ str %8, [%2, #0] @ PPCR \n\
+ ldr %0, [%1, #0] \n\
+ b 3f \n\
+2: b 1b \n\
+3: nop \n\
+ nop"
+ : "=&r" (unused)
+ : "r" (&MDCNFG), "r" (&PPCR), "0" (sd.mdcnfg),
+ "r" (sd.mdrefr), "r" (sd.mdcas[0]),
+ "r" (sd.mdcas[1]), "r" (sd.mdcas[2]), "r" (ppcr));
+ local_irq_restore(flags);
+
+ /*
+ * Now, return the SDRAM refresh back to normal.
+ */
+ sdram_update_refresh(sa11x0_freq_table[ppcr].frequency, sdram);
+
+ return 0;
+}
+
+static int __init sa1110_cpu_init(struct cpufreq_policy *policy)
+{
+ cpufreq_generic_init(policy, sa11x0_freq_table, 0);
+ return 0;
+}
+
+/* sa1110_driver needs __refdata because it must remain after init registers
+ * it with cpufreq_register_driver() */
+static struct cpufreq_driver sa1110_driver __refdata = {
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = sa1110_target,
+ .get = sa11x0_getspeed,
+ .init = sa1110_cpu_init,
+ .name = "sa1110",
+};
+
+static struct sdram_params *sa1110_find_sdram(const char *name)
+{
+ struct sdram_params *sdram;
+
+ for (sdram = sdram_tbl; sdram < sdram_tbl + ARRAY_SIZE(sdram_tbl);
+ sdram++)
+ if (strcmp(name, sdram->name) == 0)
+ return sdram;
+
+ return NULL;
+}
+
+static char sdram_name[16];
+
+static int __init sa1110_clk_init(void)
+{
+ struct sdram_params *sdram;
+ const char *name = sdram_name;
+
+ if (!cpu_is_sa1110())
+ return -ENODEV;
+
+ if (!name[0]) {
+ if (machine_is_assabet())
+ name = "TC59SM716-CL3";
+ if (machine_is_jornada720() || machine_is_h3600())
+ name = "K4S281632B-1H";
+ }
+
+ sdram = sa1110_find_sdram(name);
+ if (sdram) {
+ printk(KERN_DEBUG "SDRAM: tck: %d trcd: %d trp: %d"
+ " twr: %d refresh: %d cas_latency: %d\n",
+ sdram->tck, sdram->trcd, sdram->trp,
+ sdram->twr, sdram->refresh, sdram->cas_latency);
+
+ memcpy(&sdram_params, sdram, sizeof(sdram_params));
+
+ return cpufreq_register_driver(&sa1110_driver);
+ }
+
+ return 0;
+}
+
+module_param_string(sdram, sdram_name, sizeof(sdram_name), 0);
+arch_initcall(sa1110_clk_init);
diff --git a/drivers/cpufreq/sc520_freq.c b/drivers/cpufreq/sc520_freq.c
new file mode 100644
index 0000000000..330c8d6cf9
--- /dev/null
+++ b/drivers/cpufreq/sc520_freq.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * sc520_freq.c: cpufreq driver for the AMD Elan sc520
+ *
+ * Copyright (C) 2005 Sean Young <sean@mess.org>
+ *
+ * Based on elanfreq.c
+ *
+ * 2005-03-30: - initial revision
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+
+#include <linux/delay.h>
+#include <linux/cpufreq.h>
+#include <linux/timex.h>
+#include <linux/io.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/msr.h>
+
+#define MMCR_BASE 0xfffef000 /* The default base address */
+#define OFFS_CPUCTL 0x2 /* CPU Control Register */
+
+static __u8 __iomem *cpuctl;
+
+static struct cpufreq_frequency_table sc520_freq_table[] = {
+ {0, 0x01, 100000},
+ {0, 0x02, 133000},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu)
+{
+ u8 clockspeed_reg = *cpuctl;
+
+ switch (clockspeed_reg & 0x03) {
+ default:
+ pr_err("error: cpuctl register has unexpected value %02x\n",
+ clockspeed_reg);
+ fallthrough;
+ case 0x01:
+ return 100000;
+ case 0x02:
+ return 133000;
+ }
+}
+
+static int sc520_freq_target(struct cpufreq_policy *policy, unsigned int state)
+{
+
+ u8 clockspeed_reg;
+
+ local_irq_disable();
+
+ clockspeed_reg = *cpuctl & ~0x03;
+ *cpuctl = clockspeed_reg | sc520_freq_table[state].driver_data;
+
+ local_irq_enable();
+
+ return 0;
+}
+
+/*
+ * Module init and exit code
+ */
+
+static int sc520_freq_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpuinfo_x86 *c = &cpu_data(0);
+
+ /* capability check */
+ if (c->x86_vendor != X86_VENDOR_AMD ||
+ c->x86 != 4 || c->x86_model != 9)
+ return -ENODEV;
+
+ /* cpuinfo and default policy values */
+ policy->cpuinfo.transition_latency = 1000000; /* 1ms */
+ policy->freq_table = sc520_freq_table;
+
+ return 0;
+}
+
+
+static struct cpufreq_driver sc520_freq_driver = {
+ .get = sc520_freq_get_cpu_frequency,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = sc520_freq_target,
+ .init = sc520_freq_cpu_init,
+ .name = "sc520_freq",
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct x86_cpu_id sc520_ids[] = {
+ X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 9, NULL),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, sc520_ids);
+
+static int __init sc520_freq_init(void)
+{
+ int err;
+
+ if (!x86_match_cpu(sc520_ids))
+ return -ENODEV;
+
+ cpuctl = ioremap((unsigned long)(MMCR_BASE + OFFS_CPUCTL), 1);
+ if (!cpuctl) {
+ pr_err("sc520_freq: error: failed to remap memory\n");
+ return -ENOMEM;
+ }
+
+ err = cpufreq_register_driver(&sc520_freq_driver);
+ if (err)
+ iounmap(cpuctl);
+
+ return err;
+}
+
+
+static void __exit sc520_freq_exit(void)
+{
+ cpufreq_unregister_driver(&sc520_freq_driver);
+ iounmap(cpuctl);
+}
+
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Sean Young <sean@mess.org>");
+MODULE_DESCRIPTION("cpufreq driver for AMD's Elan sc520 CPU");
+
+module_init(sc520_freq_init);
+module_exit(sc520_freq_exit);
+
diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c
new file mode 100644
index 0000000000..028df8a5f5
--- /dev/null
+++ b/drivers/cpufreq/scmi-cpufreq.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * System Control and Power Interface (SCMI) based CPUFreq Interface driver
+ *
+ * Copyright (C) 2018-2021 ARM Ltd.
+ * Sudeep Holla <sudeep.holla@arm.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk-provider.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/energy_model.h>
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+#include <linux/scmi_protocol.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+struct scmi_data {
+ int domain_id;
+ int nr_opp;
+ struct device *cpu_dev;
+ cpumask_var_t opp_shared_cpus;
+};
+
+static struct scmi_protocol_handle *ph;
+static const struct scmi_perf_proto_ops *perf_ops;
+
+static unsigned int scmi_cpufreq_get_rate(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+ struct scmi_data *priv = policy->driver_data;
+ unsigned long rate;
+ int ret;
+
+ ret = perf_ops->freq_get(ph, priv->domain_id, &rate, false);
+ if (ret)
+ return 0;
+ return rate / 1000;
+}
+
+/*
+ * perf_ops->freq_set is not a synchronous, the actual OPP change will
+ * happen asynchronously and can get notified if the events are
+ * subscribed for by the SCMI firmware
+ */
+static int
+scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ struct scmi_data *priv = policy->driver_data;
+ u64 freq = policy->freq_table[index].frequency;
+
+ return perf_ops->freq_set(ph, priv->domain_id, freq * 1000, false);
+}
+
+static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy,
+ unsigned int target_freq)
+{
+ struct scmi_data *priv = policy->driver_data;
+
+ if (!perf_ops->freq_set(ph, priv->domain_id,
+ target_freq * 1000, true))
+ return target_freq;
+
+ return 0;
+}
+
+static int
+scmi_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask)
+{
+ int cpu, domain, tdomain;
+ struct device *tcpu_dev;
+
+ domain = perf_ops->device_domain_id(cpu_dev);
+ if (domain < 0)
+ return domain;
+
+ for_each_possible_cpu(cpu) {
+ if (cpu == cpu_dev->id)
+ continue;
+
+ tcpu_dev = get_cpu_device(cpu);
+ if (!tcpu_dev)
+ continue;
+
+ tdomain = perf_ops->device_domain_id(tcpu_dev);
+ if (tdomain == domain)
+ cpumask_set_cpu(cpu, cpumask);
+ }
+
+ return 0;
+}
+
+static int __maybe_unused
+scmi_get_cpu_power(struct device *cpu_dev, unsigned long *power,
+ unsigned long *KHz)
+{
+ enum scmi_power_scale power_scale = perf_ops->power_scale_get(ph);
+ unsigned long Hz;
+ int ret, domain;
+
+ domain = perf_ops->device_domain_id(cpu_dev);
+ if (domain < 0)
+ return domain;
+
+ /* Get the power cost of the performance domain. */
+ Hz = *KHz * 1000;
+ ret = perf_ops->est_power_get(ph, domain, &Hz, power);
+ if (ret)
+ return ret;
+
+ /* Convert the power to uW if it is mW (ignore bogoW) */
+ if (power_scale == SCMI_POWER_MILLIWATTS)
+ *power *= MICROWATT_PER_MILLIWATT;
+
+ /* The EM framework specifies the frequency in KHz. */
+ *KHz = Hz / 1000;
+
+ return 0;
+}
+
+static int scmi_cpufreq_init(struct cpufreq_policy *policy)
+{
+ int ret, nr_opp;
+ unsigned int latency;
+ struct device *cpu_dev;
+ struct scmi_data *priv;
+ struct cpufreq_frequency_table *freq_table;
+
+ cpu_dev = get_cpu_device(policy->cpu);
+ if (!cpu_dev) {
+ pr_err("failed to get cpu%d device\n", policy->cpu);
+ return -ENODEV;
+ }
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ if (!zalloc_cpumask_var(&priv->opp_shared_cpus, GFP_KERNEL)) {
+ ret = -ENOMEM;
+ goto out_free_priv;
+ }
+
+ /* Obtain CPUs that share SCMI performance controls */
+ ret = scmi_get_sharing_cpus(cpu_dev, policy->cpus);
+ if (ret) {
+ dev_warn(cpu_dev, "failed to get sharing cpumask\n");
+ goto out_free_cpumask;
+ }
+
+ /*
+ * Obtain CPUs that share performance levels.
+ * The OPP 'sharing cpus' info may come from DT through an empty opp
+ * table and opp-shared.
+ */
+ ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->opp_shared_cpus);
+ if (ret || cpumask_empty(priv->opp_shared_cpus)) {
+ /*
+ * Either opp-table is not set or no opp-shared was found.
+ * Use the CPU mask from SCMI to designate CPUs sharing an OPP
+ * table.
+ */
+ cpumask_copy(priv->opp_shared_cpus, policy->cpus);
+ }
+
+ /*
+ * A previous CPU may have marked OPPs as shared for a few CPUs, based on
+ * what OPP core provided. If the current CPU is part of those few, then
+ * there is no need to add OPPs again.
+ */
+ nr_opp = dev_pm_opp_get_opp_count(cpu_dev);
+ if (nr_opp <= 0) {
+ ret = perf_ops->device_opps_add(ph, cpu_dev);
+ if (ret) {
+ dev_warn(cpu_dev, "failed to add opps to the device\n");
+ goto out_free_cpumask;
+ }
+
+ nr_opp = dev_pm_opp_get_opp_count(cpu_dev);
+ if (nr_opp <= 0) {
+ dev_err(cpu_dev, "%s: No OPPs for this device: %d\n",
+ __func__, nr_opp);
+
+ ret = -ENODEV;
+ goto out_free_opp;
+ }
+
+ ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->opp_shared_cpus);
+ if (ret) {
+ dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
+ __func__, ret);
+
+ goto out_free_opp;
+ }
+
+ priv->nr_opp = nr_opp;
+ }
+
+ ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
+ if (ret) {
+ dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
+ goto out_free_opp;
+ }
+
+ priv->cpu_dev = cpu_dev;
+ priv->domain_id = perf_ops->device_domain_id(cpu_dev);
+
+ policy->driver_data = priv;
+ policy->freq_table = freq_table;
+
+ /* SCMI allows DVFS request for any domain from any CPU */
+ policy->dvfs_possible_from_any_cpu = true;
+
+ latency = perf_ops->transition_latency_get(ph, cpu_dev);
+ if (!latency)
+ latency = CPUFREQ_ETERNAL;
+
+ policy->cpuinfo.transition_latency = latency;
+
+ policy->fast_switch_possible =
+ perf_ops->fast_switch_possible(ph, cpu_dev);
+
+ return 0;
+
+out_free_opp:
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+
+out_free_cpumask:
+ free_cpumask_var(priv->opp_shared_cpus);
+
+out_free_priv:
+ kfree(priv);
+
+ return ret;
+}
+
+static int scmi_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ struct scmi_data *priv = policy->driver_data;
+
+ dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
+ dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
+ free_cpumask_var(priv->opp_shared_cpus);
+ kfree(priv);
+
+ return 0;
+}
+
+static void scmi_cpufreq_register_em(struct cpufreq_policy *policy)
+{
+ struct em_data_callback em_cb = EM_DATA_CB(scmi_get_cpu_power);
+ enum scmi_power_scale power_scale = perf_ops->power_scale_get(ph);
+ struct scmi_data *priv = policy->driver_data;
+ bool em_power_scale = false;
+
+ /*
+ * This callback will be called for each policy, but we don't need to
+ * register with EM every time. Despite not being part of the same
+ * policy, some CPUs may still share their perf-domains, and a CPU from
+ * another policy may already have registered with EM on behalf of CPUs
+ * of this policy.
+ */
+ if (!priv->nr_opp)
+ return;
+
+ if (power_scale == SCMI_POWER_MILLIWATTS
+ || power_scale == SCMI_POWER_MICROWATTS)
+ em_power_scale = true;
+
+ em_dev_register_perf_domain(get_cpu_device(policy->cpu), priv->nr_opp,
+ &em_cb, priv->opp_shared_cpus,
+ em_power_scale);
+}
+
+static struct cpufreq_driver scmi_cpufreq_driver = {
+ .name = "scmi",
+ .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .attr = cpufreq_generic_attr,
+ .target_index = scmi_cpufreq_set_target,
+ .fast_switch = scmi_cpufreq_fast_switch,
+ .get = scmi_cpufreq_get_rate,
+ .init = scmi_cpufreq_init,
+ .exit = scmi_cpufreq_exit,
+ .register_em = scmi_cpufreq_register_em,
+};
+
+static int scmi_cpufreq_probe(struct scmi_device *sdev)
+{
+ int ret;
+ struct device *dev = &sdev->dev;
+ const struct scmi_handle *handle;
+
+ handle = sdev->handle;
+
+ if (!handle)
+ return -ENODEV;
+
+ perf_ops = handle->devm_protocol_get(sdev, SCMI_PROTOCOL_PERF, &ph);
+ if (IS_ERR(perf_ops))
+ return PTR_ERR(perf_ops);
+
+#ifdef CONFIG_COMMON_CLK
+ /* dummy clock provider as needed by OPP if clocks property is used */
+ if (of_property_present(dev->of_node, "#clock-cells")) {
+ ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, NULL);
+ if (ret)
+ return dev_err_probe(dev, ret, "%s: registering clock provider failed\n", __func__);
+ }
+#endif
+
+ ret = cpufreq_register_driver(&scmi_cpufreq_driver);
+ if (ret) {
+ dev_err(dev, "%s: registering cpufreq failed, err: %d\n",
+ __func__, ret);
+ }
+
+ return ret;
+}
+
+static void scmi_cpufreq_remove(struct scmi_device *sdev)
+{
+ cpufreq_unregister_driver(&scmi_cpufreq_driver);
+}
+
+static const struct scmi_device_id scmi_id_table[] = {
+ { SCMI_PROTOCOL_PERF, "cpufreq" },
+ { },
+};
+MODULE_DEVICE_TABLE(scmi, scmi_id_table);
+
+static struct scmi_driver scmi_cpufreq_drv = {
+ .name = "scmi-cpufreq",
+ .probe = scmi_cpufreq_probe,
+ .remove = scmi_cpufreq_remove,
+ .id_table = scmi_id_table,
+};
+module_scmi_driver(scmi_cpufreq_drv);
+
+MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>");
+MODULE_DESCRIPTION("ARM SCMI CPUFreq interface driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c
new file mode 100644
index 0000000000..d33be56983
--- /dev/null
+++ b/drivers/cpufreq/scpi-cpufreq.c
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * System Control and Power Interface (SCPI) based CPUFreq Interface driver
+ *
+ * Copyright (C) 2015 ARM Ltd.
+ * Sudeep Holla <sudeep.holla@arm.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/scpi_protocol.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+struct scpi_data {
+ struct clk *clk;
+ struct device *cpu_dev;
+};
+
+static struct scpi_ops *scpi_ops;
+
+static unsigned int scpi_cpufreq_get_rate(unsigned int cpu)
+{
+ struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+ struct scpi_data *priv = policy->driver_data;
+ unsigned long rate = clk_get_rate(priv->clk);
+
+ return rate / 1000;
+}
+
+static int
+scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ u64 rate = policy->freq_table[index].frequency * 1000;
+ struct scpi_data *priv = policy->driver_data;
+ int ret;
+
+ ret = clk_set_rate(priv->clk, rate);
+
+ if (ret)
+ return ret;
+
+ if (clk_get_rate(priv->clk) != rate)
+ return -EIO;
+
+ return 0;
+}
+
+static int
+scpi_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask)
+{
+ int cpu, domain, tdomain;
+ struct device *tcpu_dev;
+
+ domain = scpi_ops->device_domain_id(cpu_dev);
+ if (domain < 0)
+ return domain;
+
+ for_each_possible_cpu(cpu) {
+ if (cpu == cpu_dev->id)
+ continue;
+
+ tcpu_dev = get_cpu_device(cpu);
+ if (!tcpu_dev)
+ continue;
+
+ tdomain = scpi_ops->device_domain_id(tcpu_dev);
+ if (tdomain == domain)
+ cpumask_set_cpu(cpu, cpumask);
+ }
+
+ return 0;
+}
+
+static int scpi_cpufreq_init(struct cpufreq_policy *policy)
+{
+ int ret;
+ unsigned int latency;
+ struct device *cpu_dev;
+ struct scpi_data *priv;
+ struct cpufreq_frequency_table *freq_table;
+
+ cpu_dev = get_cpu_device(policy->cpu);
+ if (!cpu_dev) {
+ pr_err("failed to get cpu%d device\n", policy->cpu);
+ return -ENODEV;
+ }
+
+ ret = scpi_ops->add_opps_to_device(cpu_dev);
+ if (ret) {
+ dev_warn(cpu_dev, "failed to add opps to the device\n");
+ return ret;
+ }
+
+ ret = scpi_get_sharing_cpus(cpu_dev, policy->cpus);
+ if (ret) {
+ dev_warn(cpu_dev, "failed to get sharing cpumask\n");
+ return ret;
+ }
+
+ ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
+ if (ret) {
+ dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
+ __func__, ret);
+ return ret;
+ }
+
+ ret = dev_pm_opp_get_opp_count(cpu_dev);
+ if (ret <= 0) {
+ dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
+ ret = -EPROBE_DEFER;
+ goto out_free_opp;
+ }
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv) {
+ ret = -ENOMEM;
+ goto out_free_opp;
+ }
+
+ ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
+ if (ret) {
+ dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
+ goto out_free_priv;
+ }
+
+ priv->cpu_dev = cpu_dev;
+ priv->clk = clk_get(cpu_dev, NULL);
+ if (IS_ERR(priv->clk)) {
+ dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d\n",
+ __func__, cpu_dev->id);
+ ret = PTR_ERR(priv->clk);
+ goto out_free_cpufreq_table;
+ }
+
+ policy->driver_data = priv;
+ policy->freq_table = freq_table;
+
+ /* scpi allows DVFS request for any domain from any CPU */
+ policy->dvfs_possible_from_any_cpu = true;
+
+ latency = scpi_ops->get_transition_latency(cpu_dev);
+ if (!latency)
+ latency = CPUFREQ_ETERNAL;
+
+ policy->cpuinfo.transition_latency = latency;
+
+ policy->fast_switch_possible = false;
+
+ return 0;
+
+out_free_cpufreq_table:
+ dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
+out_free_priv:
+ kfree(priv);
+out_free_opp:
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+
+ return ret;
+}
+
+static int scpi_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ struct scpi_data *priv = policy->driver_data;
+
+ clk_put(priv->clk);
+ dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
+ dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
+ kfree(priv);
+
+ return 0;
+}
+
+static struct cpufreq_driver scpi_cpufreq_driver = {
+ .name = "scpi-cpufreq",
+ .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .attr = cpufreq_generic_attr,
+ .get = scpi_cpufreq_get_rate,
+ .init = scpi_cpufreq_init,
+ .exit = scpi_cpufreq_exit,
+ .target_index = scpi_cpufreq_set_target,
+ .register_em = cpufreq_register_em_with_opp,
+};
+
+static int scpi_cpufreq_probe(struct platform_device *pdev)
+{
+ int ret;
+
+ scpi_ops = get_scpi_ops();
+ if (!scpi_ops)
+ return -EIO;
+
+ ret = cpufreq_register_driver(&scpi_cpufreq_driver);
+ if (ret)
+ dev_err(&pdev->dev, "%s: registering cpufreq failed, err: %d\n",
+ __func__, ret);
+ return ret;
+}
+
+static void scpi_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&scpi_cpufreq_driver);
+ scpi_ops = NULL;
+}
+
+static struct platform_driver scpi_cpufreq_platdrv = {
+ .driver = {
+ .name = "scpi-cpufreq",
+ },
+ .probe = scpi_cpufreq_probe,
+ .remove_new = scpi_cpufreq_remove,
+};
+module_platform_driver(scpi_cpufreq_platdrv);
+
+MODULE_ALIAS("platform:scpi-cpufreq");
+MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>");
+MODULE_DESCRIPTION("ARM SCPI CPUFreq interface driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/sh-cpufreq.c b/drivers/cpufreq/sh-cpufreq.c
new file mode 100644
index 0000000000..b8704232c2
--- /dev/null
+++ b/drivers/cpufreq/sh-cpufreq.c
@@ -0,0 +1,175 @@
+/*
+ * cpufreq driver for the SuperH processors.
+ *
+ * Copyright (C) 2002 - 2012 Paul Mundt
+ * Copyright (C) 2002 M. R. Brown
+ *
+ * Clock framework bits from arch/avr32/mach-at32ap/cpufreq.c
+ *
+ * Copyright (C) 2004-2007 Atmel Corporation
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#define pr_fmt(fmt) "cpufreq: " fmt
+
+#include <linux/types.h>
+#include <linux/cpufreq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/cpumask.h>
+#include <linux/cpu.h>
+#include <linux/smp.h>
+#include <linux/clk.h>
+#include <linux/percpu.h>
+#include <linux/sh_clk.h>
+
+static DEFINE_PER_CPU(struct clk, sh_cpuclk);
+
+struct cpufreq_target {
+ struct cpufreq_policy *policy;
+ unsigned int freq;
+};
+
+static unsigned int sh_cpufreq_get(unsigned int cpu)
+{
+ return (clk_get_rate(&per_cpu(sh_cpuclk, cpu)) + 500) / 1000;
+}
+
+static long __sh_cpufreq_target(void *arg)
+{
+ struct cpufreq_target *target = arg;
+ struct cpufreq_policy *policy = target->policy;
+ int cpu = policy->cpu;
+ struct clk *cpuclk = &per_cpu(sh_cpuclk, cpu);
+ struct cpufreq_freqs freqs;
+ struct device *dev;
+ long freq;
+
+ if (smp_processor_id() != cpu)
+ return -ENODEV;
+
+ dev = get_cpu_device(cpu);
+
+ /* Convert target_freq from kHz to Hz */
+ freq = clk_round_rate(cpuclk, target->freq * 1000);
+
+ if (freq < (policy->min * 1000) || freq > (policy->max * 1000))
+ return -EINVAL;
+
+ dev_dbg(dev, "requested frequency %u Hz\n", target->freq * 1000);
+
+ freqs.old = sh_cpufreq_get(cpu);
+ freqs.new = (freq + 500) / 1000;
+ freqs.flags = 0;
+
+ cpufreq_freq_transition_begin(target->policy, &freqs);
+ clk_set_rate(cpuclk, freq);
+ cpufreq_freq_transition_end(target->policy, &freqs, 0);
+
+ dev_dbg(dev, "set frequency %lu Hz\n", freq);
+ return 0;
+}
+
+/*
+ * Here we notify other drivers of the proposed change and the final change.
+ */
+static int sh_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int target_freq,
+ unsigned int relation)
+{
+ struct cpufreq_target data = { .policy = policy, .freq = target_freq };
+
+ return work_on_cpu(policy->cpu, __sh_cpufreq_target, &data);
+}
+
+static int sh_cpufreq_verify(struct cpufreq_policy_data *policy)
+{
+ struct clk *cpuclk = &per_cpu(sh_cpuclk, policy->cpu);
+ struct cpufreq_frequency_table *freq_table;
+
+ freq_table = cpuclk->nr_freqs ? cpuclk->freq_table : NULL;
+ if (freq_table)
+ return cpufreq_frequency_table_verify(policy, freq_table);
+
+ cpufreq_verify_within_cpu_limits(policy);
+
+ policy->min = (clk_round_rate(cpuclk, 1) + 500) / 1000;
+ policy->max = (clk_round_rate(cpuclk, ~0UL) + 500) / 1000;
+
+ cpufreq_verify_within_cpu_limits(policy);
+ return 0;
+}
+
+static int sh_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+ struct clk *cpuclk = &per_cpu(sh_cpuclk, cpu);
+ struct cpufreq_frequency_table *freq_table;
+ struct device *dev;
+
+ dev = get_cpu_device(cpu);
+
+ cpuclk = clk_get(dev, "cpu_clk");
+ if (IS_ERR(cpuclk)) {
+ dev_err(dev, "couldn't get CPU clk\n");
+ return PTR_ERR(cpuclk);
+ }
+
+ freq_table = cpuclk->nr_freqs ? cpuclk->freq_table : NULL;
+ if (freq_table) {
+ policy->freq_table = freq_table;
+ } else {
+ dev_notice(dev, "no frequency table found, falling back "
+ "to rate rounding.\n");
+
+ policy->min = policy->cpuinfo.min_freq =
+ (clk_round_rate(cpuclk, 1) + 500) / 1000;
+ policy->max = policy->cpuinfo.max_freq =
+ (clk_round_rate(cpuclk, ~0UL) + 500) / 1000;
+ }
+
+ return 0;
+}
+
+static int sh_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+ struct clk *cpuclk = &per_cpu(sh_cpuclk, cpu);
+
+ clk_put(cpuclk);
+
+ return 0;
+}
+
+static struct cpufreq_driver sh_cpufreq_driver = {
+ .name = "sh",
+ .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
+ .get = sh_cpufreq_get,
+ .target = sh_cpufreq_target,
+ .verify = sh_cpufreq_verify,
+ .init = sh_cpufreq_cpu_init,
+ .exit = sh_cpufreq_cpu_exit,
+ .attr = cpufreq_generic_attr,
+};
+
+static int __init sh_cpufreq_module_init(void)
+{
+ pr_notice("SuperH CPU frequency driver.\n");
+ return cpufreq_register_driver(&sh_cpufreq_driver);
+}
+
+static void __exit sh_cpufreq_module_exit(void)
+{
+ cpufreq_unregister_driver(&sh_cpufreq_driver);
+}
+
+module_init(sh_cpufreq_module_init);
+module_exit(sh_cpufreq_module_exit);
+
+MODULE_AUTHOR("Paul Mundt <lethal@linux-sh.org>");
+MODULE_DESCRIPTION("cpufreq driver for SuperH");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/sparc-us2e-cpufreq.c b/drivers/cpufreq/sparc-us2e-cpufreq.c
new file mode 100644
index 0000000000..2783d3d55f
--- /dev/null
+++ b/drivers/cpufreq/sparc-us2e-cpufreq.c
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* us2e_cpufreq.c: UltraSPARC-IIe cpu frequency support
+ *
+ * Copyright (C) 2003 David S. Miller (davem@redhat.com)
+ *
+ * Many thanks to Dominik Brodowski for fixing up the cpufreq
+ * infrastructure in order to make this driver easier to implement.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/smp.h>
+#include <linux/cpufreq.h>
+#include <linux/threads.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+
+#include <asm/asi.h>
+#include <asm/timer.h>
+
+struct us2e_freq_percpu_info {
+ struct cpufreq_frequency_table table[6];
+};
+
+/* Indexed by cpu number. */
+static struct us2e_freq_percpu_info *us2e_freq_table;
+
+#define HBIRD_MEM_CNTL0_ADDR 0x1fe0000f010UL
+#define HBIRD_ESTAR_MODE_ADDR 0x1fe0000f080UL
+
+/* UltraSPARC-IIe has five dividers: 1, 2, 4, 6, and 8. These are controlled
+ * in the ESTAR mode control register.
+ */
+#define ESTAR_MODE_DIV_1 0x0000000000000000UL
+#define ESTAR_MODE_DIV_2 0x0000000000000001UL
+#define ESTAR_MODE_DIV_4 0x0000000000000003UL
+#define ESTAR_MODE_DIV_6 0x0000000000000002UL
+#define ESTAR_MODE_DIV_8 0x0000000000000004UL
+#define ESTAR_MODE_DIV_MASK 0x0000000000000007UL
+
+#define MCTRL0_SREFRESH_ENAB 0x0000000000010000UL
+#define MCTRL0_REFR_COUNT_MASK 0x0000000000007f00UL
+#define MCTRL0_REFR_COUNT_SHIFT 8
+#define MCTRL0_REFR_INTERVAL 7800
+#define MCTRL0_REFR_CLKS_P_CNT 64
+
+static unsigned long read_hbreg(unsigned long addr)
+{
+ unsigned long ret;
+
+ __asm__ __volatile__("ldxa [%1] %2, %0"
+ : "=&r" (ret)
+ : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E));
+ return ret;
+}
+
+static void write_hbreg(unsigned long addr, unsigned long val)
+{
+ __asm__ __volatile__("stxa %0, [%1] %2\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "r" (val), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)
+ : "memory");
+ if (addr == HBIRD_ESTAR_MODE_ADDR) {
+ /* Need to wait 16 clock cycles for the PLL to lock. */
+ udelay(1);
+ }
+}
+
+static void self_refresh_ctl(int enable)
+{
+ unsigned long mctrl = read_hbreg(HBIRD_MEM_CNTL0_ADDR);
+
+ if (enable)
+ mctrl |= MCTRL0_SREFRESH_ENAB;
+ else
+ mctrl &= ~MCTRL0_SREFRESH_ENAB;
+ write_hbreg(HBIRD_MEM_CNTL0_ADDR, mctrl);
+ (void) read_hbreg(HBIRD_MEM_CNTL0_ADDR);
+}
+
+static void frob_mem_refresh(int cpu_slowing_down,
+ unsigned long clock_tick,
+ unsigned long old_divisor, unsigned long divisor)
+{
+ unsigned long old_refr_count, refr_count, mctrl;
+
+ refr_count = (clock_tick * MCTRL0_REFR_INTERVAL);
+ refr_count /= (MCTRL0_REFR_CLKS_P_CNT * divisor * 1000000000UL);
+
+ mctrl = read_hbreg(HBIRD_MEM_CNTL0_ADDR);
+ old_refr_count = (mctrl & MCTRL0_REFR_COUNT_MASK)
+ >> MCTRL0_REFR_COUNT_SHIFT;
+
+ mctrl &= ~MCTRL0_REFR_COUNT_MASK;
+ mctrl |= refr_count << MCTRL0_REFR_COUNT_SHIFT;
+ write_hbreg(HBIRD_MEM_CNTL0_ADDR, mctrl);
+ mctrl = read_hbreg(HBIRD_MEM_CNTL0_ADDR);
+
+ if (cpu_slowing_down && !(mctrl & MCTRL0_SREFRESH_ENAB)) {
+ unsigned long usecs;
+
+ /* We have to wait for both refresh counts (old
+ * and new) to go to zero.
+ */
+ usecs = (MCTRL0_REFR_CLKS_P_CNT *
+ (refr_count + old_refr_count) *
+ 1000000UL *
+ old_divisor) / clock_tick;
+ udelay(usecs + 1UL);
+ }
+}
+
+static void us2e_transition(unsigned long estar, unsigned long new_bits,
+ unsigned long clock_tick,
+ unsigned long old_divisor, unsigned long divisor)
+{
+ estar &= ~ESTAR_MODE_DIV_MASK;
+
+ /* This is based upon the state transition diagram in the IIe manual. */
+ if (old_divisor == 2 && divisor == 1) {
+ self_refresh_ctl(0);
+ write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits);
+ frob_mem_refresh(0, clock_tick, old_divisor, divisor);
+ } else if (old_divisor == 1 && divisor == 2) {
+ frob_mem_refresh(1, clock_tick, old_divisor, divisor);
+ write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits);
+ self_refresh_ctl(1);
+ } else if (old_divisor == 1 && divisor > 2) {
+ us2e_transition(estar, ESTAR_MODE_DIV_2, clock_tick,
+ 1, 2);
+ us2e_transition(estar, new_bits, clock_tick,
+ 2, divisor);
+ } else if (old_divisor > 2 && divisor == 1) {
+ us2e_transition(estar, ESTAR_MODE_DIV_2, clock_tick,
+ old_divisor, 2);
+ us2e_transition(estar, new_bits, clock_tick,
+ 2, divisor);
+ } else if (old_divisor < divisor) {
+ frob_mem_refresh(0, clock_tick, old_divisor, divisor);
+ write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits);
+ } else if (old_divisor > divisor) {
+ write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits);
+ frob_mem_refresh(1, clock_tick, old_divisor, divisor);
+ } else {
+ BUG();
+ }
+}
+
+static unsigned long index_to_estar_mode(unsigned int index)
+{
+ switch (index) {
+ case 0:
+ return ESTAR_MODE_DIV_1;
+
+ case 1:
+ return ESTAR_MODE_DIV_2;
+
+ case 2:
+ return ESTAR_MODE_DIV_4;
+
+ case 3:
+ return ESTAR_MODE_DIV_6;
+
+ case 4:
+ return ESTAR_MODE_DIV_8;
+
+ default:
+ BUG();
+ }
+}
+
+static unsigned long index_to_divisor(unsigned int index)
+{
+ switch (index) {
+ case 0:
+ return 1;
+
+ case 1:
+ return 2;
+
+ case 2:
+ return 4;
+
+ case 3:
+ return 6;
+
+ case 4:
+ return 8;
+
+ default:
+ BUG();
+ }
+}
+
+static unsigned long estar_to_divisor(unsigned long estar)
+{
+ unsigned long ret;
+
+ switch (estar & ESTAR_MODE_DIV_MASK) {
+ case ESTAR_MODE_DIV_1:
+ ret = 1;
+ break;
+ case ESTAR_MODE_DIV_2:
+ ret = 2;
+ break;
+ case ESTAR_MODE_DIV_4:
+ ret = 4;
+ break;
+ case ESTAR_MODE_DIV_6:
+ ret = 6;
+ break;
+ case ESTAR_MODE_DIV_8:
+ ret = 8;
+ break;
+ default:
+ BUG();
+ }
+
+ return ret;
+}
+
+static void __us2e_freq_get(void *arg)
+{
+ unsigned long *estar = arg;
+
+ *estar = read_hbreg(HBIRD_ESTAR_MODE_ADDR);
+}
+
+static unsigned int us2e_freq_get(unsigned int cpu)
+{
+ unsigned long clock_tick, estar;
+
+ clock_tick = sparc64_get_clock_tick(cpu) / 1000;
+ if (smp_call_function_single(cpu, __us2e_freq_get, &estar, 1))
+ return 0;
+
+ return clock_tick / estar_to_divisor(estar);
+}
+
+static void __us2e_freq_target(void *arg)
+{
+ unsigned int cpu = smp_processor_id();
+ unsigned int *index = arg;
+ unsigned long new_bits, new_freq;
+ unsigned long clock_tick, divisor, old_divisor, estar;
+
+ new_freq = clock_tick = sparc64_get_clock_tick(cpu) / 1000;
+ new_bits = index_to_estar_mode(*index);
+ divisor = index_to_divisor(*index);
+ new_freq /= divisor;
+
+ estar = read_hbreg(HBIRD_ESTAR_MODE_ADDR);
+
+ old_divisor = estar_to_divisor(estar);
+
+ if (old_divisor != divisor) {
+ us2e_transition(estar, new_bits, clock_tick * 1000,
+ old_divisor, divisor);
+ }
+}
+
+static int us2e_freq_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ unsigned int cpu = policy->cpu;
+
+ return smp_call_function_single(cpu, __us2e_freq_target, &index, 1);
+}
+
+static int us2e_freq_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+ unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000;
+ struct cpufreq_frequency_table *table =
+ &us2e_freq_table[cpu].table[0];
+
+ table[0].driver_data = 0;
+ table[0].frequency = clock_tick / 1;
+ table[1].driver_data = 1;
+ table[1].frequency = clock_tick / 2;
+ table[2].driver_data = 2;
+ table[2].frequency = clock_tick / 4;
+ table[2].driver_data = 3;
+ table[2].frequency = clock_tick / 6;
+ table[2].driver_data = 4;
+ table[2].frequency = clock_tick / 8;
+ table[2].driver_data = 5;
+ table[3].frequency = CPUFREQ_TABLE_END;
+
+ policy->cpuinfo.transition_latency = 0;
+ policy->cur = clock_tick;
+ policy->freq_table = table;
+
+ return 0;
+}
+
+static int us2e_freq_cpu_exit(struct cpufreq_policy *policy)
+{
+ us2e_freq_target(policy, 0);
+ return 0;
+}
+
+static struct cpufreq_driver cpufreq_us2e_driver = {
+ .name = "UltraSPARC-IIe",
+ .init = us2e_freq_cpu_init,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = us2e_freq_target,
+ .get = us2e_freq_get,
+ .exit = us2e_freq_cpu_exit,
+};
+
+static int __init us2e_freq_init(void)
+{
+ unsigned long manuf, impl, ver;
+ int ret;
+
+ if (tlb_type != spitfire)
+ return -ENODEV;
+
+ __asm__("rdpr %%ver, %0" : "=r" (ver));
+ manuf = ((ver >> 48) & 0xffff);
+ impl = ((ver >> 32) & 0xffff);
+
+ if (manuf == 0x17 && impl == 0x13) {
+ us2e_freq_table = kzalloc(NR_CPUS * sizeof(*us2e_freq_table),
+ GFP_KERNEL);
+ if (!us2e_freq_table)
+ return -ENOMEM;
+
+ ret = cpufreq_register_driver(&cpufreq_us2e_driver);
+ if (ret)
+ kfree(us2e_freq_table);
+
+ return ret;
+ }
+
+ return -ENODEV;
+}
+
+static void __exit us2e_freq_exit(void)
+{
+ cpufreq_unregister_driver(&cpufreq_us2e_driver);
+ kfree(us2e_freq_table);
+}
+
+MODULE_AUTHOR("David S. Miller <davem@redhat.com>");
+MODULE_DESCRIPTION("cpufreq driver for UltraSPARC-IIe");
+MODULE_LICENSE("GPL");
+
+module_init(us2e_freq_init);
+module_exit(us2e_freq_exit);
diff --git a/drivers/cpufreq/sparc-us3-cpufreq.c b/drivers/cpufreq/sparc-us3-cpufreq.c
new file mode 100644
index 0000000000..6c3657679a
--- /dev/null
+++ b/drivers/cpufreq/sparc-us3-cpufreq.c
@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* us3_cpufreq.c: UltraSPARC-III cpu frequency support
+ *
+ * Copyright (C) 2003 David S. Miller (davem@redhat.com)
+ *
+ * Many thanks to Dominik Brodowski for fixing up the cpufreq
+ * infrastructure in order to make this driver easier to implement.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/smp.h>
+#include <linux/cpufreq.h>
+#include <linux/threads.h>
+#include <linux/slab.h>
+#include <linux/init.h>
+
+#include <asm/head.h>
+#include <asm/timer.h>
+
+struct us3_freq_percpu_info {
+ struct cpufreq_frequency_table table[4];
+};
+
+/* Indexed by cpu number. */
+static struct us3_freq_percpu_info *us3_freq_table;
+
+/* UltraSPARC-III has three dividers: 1, 2, and 32. These are controlled
+ * in the Safari config register.
+ */
+#define SAFARI_CFG_DIV_1 0x0000000000000000UL
+#define SAFARI_CFG_DIV_2 0x0000000040000000UL
+#define SAFARI_CFG_DIV_32 0x0000000080000000UL
+#define SAFARI_CFG_DIV_MASK 0x00000000C0000000UL
+
+static void read_safari_cfg(void *arg)
+{
+ unsigned long ret, *val = arg;
+
+ __asm__ __volatile__("ldxa [%%g0] %1, %0"
+ : "=&r" (ret)
+ : "i" (ASI_SAFARI_CONFIG));
+ *val = ret;
+}
+
+static void update_safari_cfg(void *arg)
+{
+ unsigned long reg, *new_bits = arg;
+
+ read_safari_cfg(&reg);
+ reg &= ~SAFARI_CFG_DIV_MASK;
+ reg |= *new_bits;
+
+ __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "r" (reg), "i" (ASI_SAFARI_CONFIG)
+ : "memory");
+}
+
+static unsigned long get_current_freq(unsigned int cpu, unsigned long safari_cfg)
+{
+ unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000;
+ unsigned long ret;
+
+ switch (safari_cfg & SAFARI_CFG_DIV_MASK) {
+ case SAFARI_CFG_DIV_1:
+ ret = clock_tick / 1;
+ break;
+ case SAFARI_CFG_DIV_2:
+ ret = clock_tick / 2;
+ break;
+ case SAFARI_CFG_DIV_32:
+ ret = clock_tick / 32;
+ break;
+ default:
+ BUG();
+ }
+
+ return ret;
+}
+
+static unsigned int us3_freq_get(unsigned int cpu)
+{
+ unsigned long reg;
+
+ if (smp_call_function_single(cpu, read_safari_cfg, &reg, 1))
+ return 0;
+ return get_current_freq(cpu, reg);
+}
+
+static int us3_freq_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ unsigned int cpu = policy->cpu;
+ unsigned long new_bits, new_freq;
+
+ new_freq = sparc64_get_clock_tick(cpu) / 1000;
+ switch (index) {
+ case 0:
+ new_bits = SAFARI_CFG_DIV_1;
+ new_freq /= 1;
+ break;
+ case 1:
+ new_bits = SAFARI_CFG_DIV_2;
+ new_freq /= 2;
+ break;
+ case 2:
+ new_bits = SAFARI_CFG_DIV_32;
+ new_freq /= 32;
+ break;
+
+ default:
+ BUG();
+ }
+
+ return smp_call_function_single(cpu, update_safari_cfg, &new_bits, 1);
+}
+
+static int us3_freq_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+ unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000;
+ struct cpufreq_frequency_table *table =
+ &us3_freq_table[cpu].table[0];
+
+ table[0].driver_data = 0;
+ table[0].frequency = clock_tick / 1;
+ table[1].driver_data = 1;
+ table[1].frequency = clock_tick / 2;
+ table[2].driver_data = 2;
+ table[2].frequency = clock_tick / 32;
+ table[3].driver_data = 0;
+ table[3].frequency = CPUFREQ_TABLE_END;
+
+ policy->cpuinfo.transition_latency = 0;
+ policy->cur = clock_tick;
+ policy->freq_table = table;
+
+ return 0;
+}
+
+static int us3_freq_cpu_exit(struct cpufreq_policy *policy)
+{
+ us3_freq_target(policy, 0);
+ return 0;
+}
+
+static struct cpufreq_driver cpufreq_us3_driver = {
+ .name = "UltraSPARC-III",
+ .init = us3_freq_cpu_init,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = us3_freq_target,
+ .get = us3_freq_get,
+ .exit = us3_freq_cpu_exit,
+};
+
+static int __init us3_freq_init(void)
+{
+ unsigned long manuf, impl, ver;
+ int ret;
+
+ if (tlb_type != cheetah && tlb_type != cheetah_plus)
+ return -ENODEV;
+
+ __asm__("rdpr %%ver, %0" : "=r" (ver));
+ manuf = ((ver >> 48) & 0xffff);
+ impl = ((ver >> 32) & 0xffff);
+
+ if (manuf == CHEETAH_MANUF &&
+ (impl == CHEETAH_IMPL ||
+ impl == CHEETAH_PLUS_IMPL ||
+ impl == JAGUAR_IMPL ||
+ impl == PANTHER_IMPL)) {
+ us3_freq_table = kzalloc(NR_CPUS * sizeof(*us3_freq_table),
+ GFP_KERNEL);
+ if (!us3_freq_table)
+ return -ENOMEM;
+
+ ret = cpufreq_register_driver(&cpufreq_us3_driver);
+ if (ret)
+ kfree(us3_freq_table);
+
+ return ret;
+ }
+
+ return -ENODEV;
+}
+
+static void __exit us3_freq_exit(void)
+{
+ cpufreq_unregister_driver(&cpufreq_us3_driver);
+ kfree(us3_freq_table);
+}
+
+MODULE_AUTHOR("David S. Miller <davem@redhat.com>");
+MODULE_DESCRIPTION("cpufreq driver for UltraSPARC-III");
+MODULE_LICENSE("GPL");
+
+module_init(us3_freq_init);
+module_exit(us3_freq_exit);
diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c
new file mode 100644
index 0000000000..78b875db6b
--- /dev/null
+++ b/drivers/cpufreq/spear-cpufreq.c
@@ -0,0 +1,247 @@
+/*
+ * drivers/cpufreq/spear-cpufreq.c
+ *
+ * CPU Frequency Scaling for SPEAr platform
+ *
+ * Copyright (C) 2012 ST Microelectronics
+ * Deepak Sikri <deepak.sikri@st.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+/* SPEAr CPUFreq driver data structure */
+static struct {
+ struct clk *clk;
+ unsigned int transition_latency;
+ struct cpufreq_frequency_table *freq_tbl;
+ u32 cnt;
+} spear_cpufreq;
+
+static struct clk *spear1340_cpu_get_possible_parent(unsigned long newfreq)
+{
+ struct clk *sys_pclk;
+ int pclk;
+ /*
+ * In SPEAr1340, cpu clk's parent sys clk can take input from
+ * following sources
+ */
+ static const char * const sys_clk_src[] = {
+ "sys_syn_clk",
+ "pll1_clk",
+ "pll2_clk",
+ "pll3_clk",
+ };
+
+ /*
+ * As sys clk can have multiple source with their own range
+ * limitation so we choose possible sources accordingly
+ */
+ if (newfreq <= 300000000)
+ pclk = 0; /* src is sys_syn_clk */
+ else if (newfreq > 300000000 && newfreq <= 500000000)
+ pclk = 3; /* src is pll3_clk */
+ else if (newfreq == 600000000)
+ pclk = 1; /* src is pll1_clk */
+ else
+ return ERR_PTR(-EINVAL);
+
+ /* Get parent to sys clock */
+ sys_pclk = clk_get(NULL, sys_clk_src[pclk]);
+ if (IS_ERR(sys_pclk))
+ pr_err("Failed to get %s clock\n", sys_clk_src[pclk]);
+
+ return sys_pclk;
+}
+
+/*
+ * In SPEAr1340, we cannot use newfreq directly because we need to actually
+ * access a source clock (clk) which might not be ancestor of cpu at present.
+ * Hence in SPEAr1340 we would operate on source clock directly before switching
+ * cpu clock to it.
+ */
+static int spear1340_set_cpu_rate(struct clk *sys_pclk, unsigned long newfreq)
+{
+ struct clk *sys_clk;
+ int ret = 0;
+
+ sys_clk = clk_get_parent(spear_cpufreq.clk);
+ if (IS_ERR(sys_clk)) {
+ pr_err("failed to get cpu's parent (sys) clock\n");
+ return PTR_ERR(sys_clk);
+ }
+
+ /* Set the rate of the source clock before changing the parent */
+ ret = clk_set_rate(sys_pclk, newfreq);
+ if (ret) {
+ pr_err("Failed to set sys clk rate to %lu\n", newfreq);
+ return ret;
+ }
+
+ ret = clk_set_parent(sys_clk, sys_pclk);
+ if (ret) {
+ pr_err("Failed to set sys clk parent\n");
+ return ret;
+ }
+
+ return 0;
+}
+
+static int spear_cpufreq_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ long newfreq;
+ struct clk *srcclk;
+ int ret, mult = 1;
+
+ newfreq = spear_cpufreq.freq_tbl[index].frequency * 1000;
+
+ if (of_machine_is_compatible("st,spear1340")) {
+ /*
+ * SPEAr1340 is special in the sense that due to the possibility
+ * of multiple clock sources for cpu clk's parent we can have
+ * different clock source for different frequency of cpu clk.
+ * Hence we need to choose one from amongst these possible clock
+ * sources.
+ */
+ srcclk = spear1340_cpu_get_possible_parent(newfreq);
+ if (IS_ERR(srcclk)) {
+ pr_err("Failed to get src clk\n");
+ return PTR_ERR(srcclk);
+ }
+
+ /* SPEAr1340: src clk is always 2 * intended cpu clk */
+ mult = 2;
+ } else {
+ /*
+ * src clock to be altered is ancestor of cpu clock. Hence we
+ * can directly work on cpu clk
+ */
+ srcclk = spear_cpufreq.clk;
+ }
+
+ newfreq = clk_round_rate(srcclk, newfreq * mult);
+ if (newfreq <= 0) {
+ pr_err("clk_round_rate failed for cpu src clock\n");
+ return newfreq;
+ }
+
+ if (mult == 2)
+ ret = spear1340_set_cpu_rate(srcclk, newfreq);
+ else
+ ret = clk_set_rate(spear_cpufreq.clk, newfreq);
+
+ if (ret)
+ pr_err("CPU Freq: cpu clk_set_rate failed: %d\n", ret);
+
+ return ret;
+}
+
+static int spear_cpufreq_init(struct cpufreq_policy *policy)
+{
+ policy->clk = spear_cpufreq.clk;
+ cpufreq_generic_init(policy, spear_cpufreq.freq_tbl,
+ spear_cpufreq.transition_latency);
+ return 0;
+}
+
+static struct cpufreq_driver spear_cpufreq_driver = {
+ .name = "cpufreq-spear",
+ .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = spear_cpufreq_target,
+ .get = cpufreq_generic_get,
+ .init = spear_cpufreq_init,
+ .attr = cpufreq_generic_attr,
+};
+
+static int spear_cpufreq_probe(struct platform_device *pdev)
+{
+ struct device_node *np;
+ const struct property *prop;
+ struct cpufreq_frequency_table *freq_tbl;
+ const __be32 *val;
+ int cnt, i, ret;
+
+ np = of_cpu_device_node_get(0);
+ if (!np) {
+ pr_err("No cpu node found\n");
+ return -ENODEV;
+ }
+
+ if (of_property_read_u32(np, "clock-latency",
+ &spear_cpufreq.transition_latency))
+ spear_cpufreq.transition_latency = CPUFREQ_ETERNAL;
+
+ prop = of_find_property(np, "cpufreq_tbl", NULL);
+ if (!prop || !prop->value) {
+ pr_err("Invalid cpufreq_tbl\n");
+ ret = -ENODEV;
+ goto out_put_node;
+ }
+
+ cnt = prop->length / sizeof(u32);
+ val = prop->value;
+
+ freq_tbl = kcalloc(cnt + 1, sizeof(*freq_tbl), GFP_KERNEL);
+ if (!freq_tbl) {
+ ret = -ENOMEM;
+ goto out_put_node;
+ }
+
+ for (i = 0; i < cnt; i++)
+ freq_tbl[i].frequency = be32_to_cpup(val++);
+
+ freq_tbl[i].frequency = CPUFREQ_TABLE_END;
+
+ spear_cpufreq.freq_tbl = freq_tbl;
+
+ of_node_put(np);
+
+ spear_cpufreq.clk = clk_get(NULL, "cpu_clk");
+ if (IS_ERR(spear_cpufreq.clk)) {
+ pr_err("Unable to get CPU clock\n");
+ ret = PTR_ERR(spear_cpufreq.clk);
+ goto out_put_mem;
+ }
+
+ ret = cpufreq_register_driver(&spear_cpufreq_driver);
+ if (!ret)
+ return 0;
+
+ pr_err("failed register driver: %d\n", ret);
+ clk_put(spear_cpufreq.clk);
+
+out_put_mem:
+ kfree(freq_tbl);
+ return ret;
+
+out_put_node:
+ of_node_put(np);
+ return ret;
+}
+
+static struct platform_driver spear_cpufreq_platdrv = {
+ .driver = {
+ .name = "spear-cpufreq",
+ },
+ .probe = spear_cpufreq_probe,
+};
+module_platform_driver(spear_cpufreq_platdrv);
+
+MODULE_AUTHOR("Deepak Sikri <deepak.sikri@st.com>");
+MODULE_DESCRIPTION("SPEAr CPUFreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/speedstep-centrino.c b/drivers/cpufreq/speedstep-centrino.c
new file mode 100644
index 0000000000..75b10ecdb6
--- /dev/null
+++ b/drivers/cpufreq/speedstep-centrino.c
@@ -0,0 +1,561 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * cpufreq driver for Enhanced SpeedStep, as found in Intel's Pentium
+ * M (part of the Centrino chipset).
+ *
+ * Since the original Pentium M, most new Intel CPUs support Enhanced
+ * SpeedStep.
+ *
+ * Despite the "SpeedStep" in the name, this is almost entirely unlike
+ * traditional SpeedStep.
+ *
+ * Modelled on speedstep.c
+ *
+ * Copyright (C) 2003 Jeremy Fitzhardinge <jeremy@goop.org>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/sched.h> /* current */
+#include <linux/delay.h>
+#include <linux/compiler.h>
+#include <linux/gfp.h>
+
+#include <asm/msr.h>
+#include <asm/processor.h>
+#include <asm/cpufeature.h>
+#include <asm/cpu_device_id.h>
+
+#define MAINTAINER "linux-pm@vger.kernel.org"
+
+#define INTEL_MSR_RANGE (0xffff)
+
+struct cpu_id
+{
+ __u8 x86; /* CPU family */
+ __u8 x86_model; /* model */
+ __u8 x86_stepping; /* stepping */
+};
+
+enum {
+ CPU_BANIAS,
+ CPU_DOTHAN_A1,
+ CPU_DOTHAN_A2,
+ CPU_DOTHAN_B0,
+ CPU_MP4HT_D0,
+ CPU_MP4HT_E0,
+};
+
+static const struct cpu_id cpu_ids[] = {
+ [CPU_BANIAS] = { 6, 9, 5 },
+ [CPU_DOTHAN_A1] = { 6, 13, 1 },
+ [CPU_DOTHAN_A2] = { 6, 13, 2 },
+ [CPU_DOTHAN_B0] = { 6, 13, 6 },
+ [CPU_MP4HT_D0] = {15, 3, 4 },
+ [CPU_MP4HT_E0] = {15, 4, 1 },
+};
+#define N_IDS ARRAY_SIZE(cpu_ids)
+
+struct cpu_model
+{
+ const struct cpu_id *cpu_id;
+ const char *model_name;
+ unsigned max_freq; /* max clock in kHz */
+
+ struct cpufreq_frequency_table *op_points; /* clock/voltage pairs */
+};
+static int centrino_verify_cpu_id(const struct cpuinfo_x86 *c,
+ const struct cpu_id *x);
+
+/* Operating points for current CPU */
+static DEFINE_PER_CPU(struct cpu_model *, centrino_model);
+static DEFINE_PER_CPU(const struct cpu_id *, centrino_cpu);
+
+static struct cpufreq_driver centrino_driver;
+
+#ifdef CONFIG_X86_SPEEDSTEP_CENTRINO_TABLE
+
+/* Computes the correct form for IA32_PERF_CTL MSR for a particular
+ frequency/voltage operating point; frequency in MHz, volts in mV.
+ This is stored as "driver_data" in the structure. */
+#define OP(mhz, mv) \
+ { \
+ .frequency = (mhz) * 1000, \
+ .driver_data = (((mhz)/100) << 8) | ((mv - 700) / 16) \
+ }
+
+/*
+ * These voltage tables were derived from the Intel Pentium M
+ * datasheet, document 25261202.pdf, Table 5. I have verified they
+ * are consistent with my IBM ThinkPad X31, which has a 1.3GHz Pentium
+ * M.
+ */
+
+/* Ultra Low Voltage Intel Pentium M processor 900MHz (Banias) */
+static struct cpufreq_frequency_table banias_900[] =
+{
+ OP(600, 844),
+ OP(800, 988),
+ OP(900, 1004),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+/* Ultra Low Voltage Intel Pentium M processor 1000MHz (Banias) */
+static struct cpufreq_frequency_table banias_1000[] =
+{
+ OP(600, 844),
+ OP(800, 972),
+ OP(900, 988),
+ OP(1000, 1004),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+/* Low Voltage Intel Pentium M processor 1.10GHz (Banias) */
+static struct cpufreq_frequency_table banias_1100[] =
+{
+ OP( 600, 956),
+ OP( 800, 1020),
+ OP( 900, 1100),
+ OP(1000, 1164),
+ OP(1100, 1180),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+
+/* Low Voltage Intel Pentium M processor 1.20GHz (Banias) */
+static struct cpufreq_frequency_table banias_1200[] =
+{
+ OP( 600, 956),
+ OP( 800, 1004),
+ OP( 900, 1020),
+ OP(1000, 1100),
+ OP(1100, 1164),
+ OP(1200, 1180),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+/* Intel Pentium M processor 1.30GHz (Banias) */
+static struct cpufreq_frequency_table banias_1300[] =
+{
+ OP( 600, 956),
+ OP( 800, 1260),
+ OP(1000, 1292),
+ OP(1200, 1356),
+ OP(1300, 1388),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+/* Intel Pentium M processor 1.40GHz (Banias) */
+static struct cpufreq_frequency_table banias_1400[] =
+{
+ OP( 600, 956),
+ OP( 800, 1180),
+ OP(1000, 1308),
+ OP(1200, 1436),
+ OP(1400, 1484),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+/* Intel Pentium M processor 1.50GHz (Banias) */
+static struct cpufreq_frequency_table banias_1500[] =
+{
+ OP( 600, 956),
+ OP( 800, 1116),
+ OP(1000, 1228),
+ OP(1200, 1356),
+ OP(1400, 1452),
+ OP(1500, 1484),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+/* Intel Pentium M processor 1.60GHz (Banias) */
+static struct cpufreq_frequency_table banias_1600[] =
+{
+ OP( 600, 956),
+ OP( 800, 1036),
+ OP(1000, 1164),
+ OP(1200, 1276),
+ OP(1400, 1420),
+ OP(1600, 1484),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+
+/* Intel Pentium M processor 1.70GHz (Banias) */
+static struct cpufreq_frequency_table banias_1700[] =
+{
+ OP( 600, 956),
+ OP( 800, 1004),
+ OP(1000, 1116),
+ OP(1200, 1228),
+ OP(1400, 1308),
+ OP(1700, 1484),
+ { .frequency = CPUFREQ_TABLE_END }
+};
+#undef OP
+
+#define _BANIAS(cpuid, max, name) \
+{ .cpu_id = cpuid, \
+ .model_name = "Intel(R) Pentium(R) M processor " name "MHz", \
+ .max_freq = (max)*1000, \
+ .op_points = banias_##max, \
+}
+#define BANIAS(max) _BANIAS(&cpu_ids[CPU_BANIAS], max, #max)
+
+/* CPU models, their operating frequency range, and freq/voltage
+ operating points */
+static struct cpu_model models[] =
+{
+ _BANIAS(&cpu_ids[CPU_BANIAS], 900, " 900"),
+ BANIAS(1000),
+ BANIAS(1100),
+ BANIAS(1200),
+ BANIAS(1300),
+ BANIAS(1400),
+ BANIAS(1500),
+ BANIAS(1600),
+ BANIAS(1700),
+
+ /* NULL model_name is a wildcard */
+ { &cpu_ids[CPU_DOTHAN_A1], NULL, 0, NULL },
+ { &cpu_ids[CPU_DOTHAN_A2], NULL, 0, NULL },
+ { &cpu_ids[CPU_DOTHAN_B0], NULL, 0, NULL },
+ { &cpu_ids[CPU_MP4HT_D0], NULL, 0, NULL },
+ { &cpu_ids[CPU_MP4HT_E0], NULL, 0, NULL },
+
+ { NULL, }
+};
+#undef _BANIAS
+#undef BANIAS
+
+static int centrino_cpu_init_table(struct cpufreq_policy *policy)
+{
+ struct cpuinfo_x86 *cpu = &cpu_data(policy->cpu);
+ struct cpu_model *model;
+
+ for(model = models; model->cpu_id != NULL; model++)
+ if (centrino_verify_cpu_id(cpu, model->cpu_id) &&
+ (model->model_name == NULL ||
+ strcmp(cpu->x86_model_id, model->model_name) == 0))
+ break;
+
+ if (model->cpu_id == NULL) {
+ /* No match at all */
+ pr_debug("no support for CPU model \"%s\": "
+ "send /proc/cpuinfo to " MAINTAINER "\n",
+ cpu->x86_model_id);
+ return -ENOENT;
+ }
+
+ if (model->op_points == NULL) {
+ /* Matched a non-match */
+ pr_debug("no table support for CPU model \"%s\"\n",
+ cpu->x86_model_id);
+ pr_debug("try using the acpi-cpufreq driver\n");
+ return -ENOENT;
+ }
+
+ per_cpu(centrino_model, policy->cpu) = model;
+
+ pr_debug("found \"%s\": max frequency: %dkHz\n",
+ model->model_name, model->max_freq);
+
+ return 0;
+}
+
+#else
+static inline int centrino_cpu_init_table(struct cpufreq_policy *policy)
+{
+ return -ENODEV;
+}
+#endif /* CONFIG_X86_SPEEDSTEP_CENTRINO_TABLE */
+
+static int centrino_verify_cpu_id(const struct cpuinfo_x86 *c,
+ const struct cpu_id *x)
+{
+ if ((c->x86 == x->x86) &&
+ (c->x86_model == x->x86_model) &&
+ (c->x86_stepping == x->x86_stepping))
+ return 1;
+ return 0;
+}
+
+/* To be called only after centrino_model is initialized */
+static unsigned extract_clock(unsigned msr, unsigned int cpu, int failsafe)
+{
+ int i;
+
+ /*
+ * Extract clock in kHz from PERF_CTL value
+ * for centrino, as some DSDTs are buggy.
+ * Ideally, this can be done using the acpi_data structure.
+ */
+ if ((per_cpu(centrino_cpu, cpu) == &cpu_ids[CPU_BANIAS]) ||
+ (per_cpu(centrino_cpu, cpu) == &cpu_ids[CPU_DOTHAN_A1]) ||
+ (per_cpu(centrino_cpu, cpu) == &cpu_ids[CPU_DOTHAN_B0])) {
+ msr = (msr >> 8) & 0xff;
+ return msr * 100000;
+ }
+
+ if ((!per_cpu(centrino_model, cpu)) ||
+ (!per_cpu(centrino_model, cpu)->op_points))
+ return 0;
+
+ msr &= 0xffff;
+ for (i = 0;
+ per_cpu(centrino_model, cpu)->op_points[i].frequency
+ != CPUFREQ_TABLE_END;
+ i++) {
+ if (msr == per_cpu(centrino_model, cpu)->op_points[i].driver_data)
+ return per_cpu(centrino_model, cpu)->
+ op_points[i].frequency;
+ }
+ if (failsafe)
+ return per_cpu(centrino_model, cpu)->op_points[i-1].frequency;
+ else
+ return 0;
+}
+
+/* Return the current CPU frequency in kHz */
+static unsigned int get_cur_freq(unsigned int cpu)
+{
+ unsigned l, h;
+ unsigned clock_freq;
+
+ rdmsr_on_cpu(cpu, MSR_IA32_PERF_STATUS, &l, &h);
+ clock_freq = extract_clock(l, cpu, 0);
+
+ if (unlikely(clock_freq == 0)) {
+ /*
+ * On some CPUs, we can see transient MSR values (which are
+ * not present in _PSS), while CPU is doing some automatic
+ * P-state transition (like TM2). Get the last freq set
+ * in PERF_CTL.
+ */
+ rdmsr_on_cpu(cpu, MSR_IA32_PERF_CTL, &l, &h);
+ clock_freq = extract_clock(l, cpu, 1);
+ }
+ return clock_freq;
+}
+
+
+static int centrino_cpu_init(struct cpufreq_policy *policy)
+{
+ struct cpuinfo_x86 *cpu = &cpu_data(policy->cpu);
+ unsigned l, h;
+ int i;
+
+ /* Only Intel makes Enhanced Speedstep-capable CPUs */
+ if (cpu->x86_vendor != X86_VENDOR_INTEL ||
+ !cpu_has(cpu, X86_FEATURE_EST))
+ return -ENODEV;
+
+ if (cpu_has(cpu, X86_FEATURE_CONSTANT_TSC))
+ centrino_driver.flags |= CPUFREQ_CONST_LOOPS;
+
+ if (policy->cpu != 0)
+ return -ENODEV;
+
+ for (i = 0; i < N_IDS; i++)
+ if (centrino_verify_cpu_id(cpu, &cpu_ids[i]))
+ break;
+
+ if (i != N_IDS)
+ per_cpu(centrino_cpu, policy->cpu) = &cpu_ids[i];
+
+ if (!per_cpu(centrino_cpu, policy->cpu)) {
+ pr_debug("found unsupported CPU with "
+ "Enhanced SpeedStep: send /proc/cpuinfo to "
+ MAINTAINER "\n");
+ return -ENODEV;
+ }
+
+ if (centrino_cpu_init_table(policy))
+ return -ENODEV;
+
+ /* Check to see if Enhanced SpeedStep is enabled, and try to
+ enable it if not. */
+ rdmsr(MSR_IA32_MISC_ENABLE, l, h);
+
+ if (!(l & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
+ l |= MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP;
+ pr_debug("trying to enable Enhanced SpeedStep (%x)\n", l);
+ wrmsr(MSR_IA32_MISC_ENABLE, l, h);
+
+ /* check to see if it stuck */
+ rdmsr(MSR_IA32_MISC_ENABLE, l, h);
+ if (!(l & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
+ pr_info("couldn't enable Enhanced SpeedStep\n");
+ return -ENODEV;
+ }
+ }
+
+ policy->cpuinfo.transition_latency = 10000;
+ /* 10uS transition latency */
+ policy->freq_table = per_cpu(centrino_model, policy->cpu)->op_points;
+
+ return 0;
+}
+
+static int centrino_cpu_exit(struct cpufreq_policy *policy)
+{
+ unsigned int cpu = policy->cpu;
+
+ if (!per_cpu(centrino_model, cpu))
+ return -ENODEV;
+
+ per_cpu(centrino_model, cpu) = NULL;
+
+ return 0;
+}
+
+/**
+ * centrino_target - set a new CPUFreq policy
+ * @policy: new policy
+ * @index: index of target frequency
+ *
+ * Sets a new CPUFreq policy.
+ */
+static int centrino_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ unsigned int msr, oldmsr = 0, h = 0, cpu = policy->cpu;
+ int retval = 0;
+ unsigned int j, first_cpu;
+ struct cpufreq_frequency_table *op_points;
+ cpumask_var_t covered_cpus;
+
+ if (unlikely(!zalloc_cpumask_var(&covered_cpus, GFP_KERNEL)))
+ return -ENOMEM;
+
+ if (unlikely(per_cpu(centrino_model, cpu) == NULL)) {
+ retval = -ENODEV;
+ goto out;
+ }
+
+ first_cpu = 1;
+ op_points = &per_cpu(centrino_model, cpu)->op_points[index];
+ for_each_cpu(j, policy->cpus) {
+ int good_cpu;
+
+ /*
+ * Support for SMP systems.
+ * Make sure we are running on CPU that wants to change freq
+ */
+ if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY)
+ good_cpu = cpumask_any_and(policy->cpus,
+ cpu_online_mask);
+ else
+ good_cpu = j;
+
+ if (good_cpu >= nr_cpu_ids) {
+ pr_debug("couldn't limit to CPUs in this domain\n");
+ retval = -EAGAIN;
+ if (first_cpu) {
+ /* We haven't started the transition yet. */
+ goto out;
+ }
+ break;
+ }
+
+ msr = op_points->driver_data;
+
+ if (first_cpu) {
+ rdmsr_on_cpu(good_cpu, MSR_IA32_PERF_CTL, &oldmsr, &h);
+ if (msr == (oldmsr & 0xffff)) {
+ pr_debug("no change needed - msr was and needs "
+ "to be %x\n", oldmsr);
+ retval = 0;
+ goto out;
+ }
+
+ first_cpu = 0;
+ /* all but 16 LSB are reserved, treat them with care */
+ oldmsr &= ~0xffff;
+ msr &= 0xffff;
+ oldmsr |= msr;
+ }
+
+ wrmsr_on_cpu(good_cpu, MSR_IA32_PERF_CTL, oldmsr, h);
+ if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY)
+ break;
+
+ cpumask_set_cpu(j, covered_cpus);
+ }
+
+ if (unlikely(retval)) {
+ /*
+ * We have failed halfway through the frequency change.
+ * We have sent callbacks to policy->cpus and
+ * MSRs have already been written on coverd_cpus.
+ * Best effort undo..
+ */
+
+ for_each_cpu(j, covered_cpus)
+ wrmsr_on_cpu(j, MSR_IA32_PERF_CTL, oldmsr, h);
+ }
+ retval = 0;
+
+out:
+ free_cpumask_var(covered_cpus);
+ return retval;
+}
+
+static struct cpufreq_driver centrino_driver = {
+ .name = "centrino", /* should be speedstep-centrino,
+ but there's a 16 char limit */
+ .init = centrino_cpu_init,
+ .exit = centrino_cpu_exit,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = centrino_target,
+ .get = get_cur_freq,
+ .attr = cpufreq_generic_attr,
+};
+
+/*
+ * This doesn't replace the detailed checks above because
+ * the generic CPU IDs don't have a way to match for steppings
+ * or ASCII model IDs.
+ */
+static const struct x86_cpu_id centrino_ids[] = {
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, 9, X86_FEATURE_EST, NULL),
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, 13, X86_FEATURE_EST, NULL),
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 15, 3, X86_FEATURE_EST, NULL),
+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 15, 4, X86_FEATURE_EST, NULL),
+ {}
+};
+
+/**
+ * centrino_init - initializes the Enhanced SpeedStep CPUFreq driver
+ *
+ * Initializes the Enhanced SpeedStep support. Returns -ENODEV on
+ * unsupported devices, -ENOENT if there's no voltage table for this
+ * particular CPU model, -EINVAL on problems during initiatization,
+ * and zero on success.
+ *
+ * This is quite picky. Not only does the CPU have to advertise the
+ * "est" flag in the cpuid capability flags, we look for a specific
+ * CPU model and stepping, and we need to have the exact model name in
+ * our voltage tables. That is, be paranoid about not releasing
+ * someone's valuable magic smoke.
+ */
+static int __init centrino_init(void)
+{
+ if (!x86_match_cpu(centrino_ids))
+ return -ENODEV;
+ return cpufreq_register_driver(&centrino_driver);
+}
+
+static void __exit centrino_exit(void)
+{
+ cpufreq_unregister_driver(&centrino_driver);
+}
+
+MODULE_AUTHOR ("Jeremy Fitzhardinge <jeremy@goop.org>");
+MODULE_DESCRIPTION ("Enhanced SpeedStep driver for Intel Pentium M processors.");
+MODULE_LICENSE ("GPL");
+
+late_initcall(centrino_init);
+module_exit(centrino_exit);
diff --git a/drivers/cpufreq/speedstep-ich.c b/drivers/cpufreq/speedstep-ich.c
new file mode 100644
index 0000000000..f2076d72bf
--- /dev/null
+++ b/drivers/cpufreq/speedstep-ich.c
@@ -0,0 +1,386 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * (C) 2001 Dave Jones, Arjan van de ven.
+ * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ *
+ * Based upon reverse engineered information, and on Intel documentation
+ * for chipsets ICH2-M and ICH3-M.
+ *
+ * Many thanks to Ducrot Bruno for finding and fixing the last
+ * "missing link" for ICH2-M/ICH3-M support, and to Thomas Winkler
+ * for extensive testing.
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+
+/*********************************************************************
+ * SPEEDSTEP - DEFINITIONS *
+ *********************************************************************/
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/pci.h>
+#include <linux/sched.h>
+
+#include <asm/cpu_device_id.h>
+
+#include "speedstep-lib.h"
+
+
+/* speedstep_chipset:
+ * It is necessary to know which chipset is used. As accesses to
+ * this device occur at various places in this module, we need a
+ * static struct pci_dev * pointing to that device.
+ */
+static struct pci_dev *speedstep_chipset_dev;
+
+
+/* speedstep_processor
+ */
+static enum speedstep_processor speedstep_processor;
+
+static u32 pmbase;
+
+/*
+ * There are only two frequency states for each processor. Values
+ * are in kHz for the time being.
+ */
+static struct cpufreq_frequency_table speedstep_freqs[] = {
+ {0, SPEEDSTEP_HIGH, 0},
+ {0, SPEEDSTEP_LOW, 0},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+
+/**
+ * speedstep_find_register - read the PMBASE address
+ *
+ * Returns: -ENODEV if no register could be found
+ */
+static int speedstep_find_register(void)
+{
+ if (!speedstep_chipset_dev)
+ return -ENODEV;
+
+ /* get PMBASE */
+ pci_read_config_dword(speedstep_chipset_dev, 0x40, &pmbase);
+ if (!(pmbase & 0x01)) {
+ pr_err("could not find speedstep register\n");
+ return -ENODEV;
+ }
+
+ pmbase &= 0xFFFFFFFE;
+ if (!pmbase) {
+ pr_err("could not find speedstep register\n");
+ return -ENODEV;
+ }
+
+ pr_debug("pmbase is 0x%x\n", pmbase);
+ return 0;
+}
+
+/**
+ * speedstep_set_state - set the SpeedStep state
+ * @state: new processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH)
+ *
+ * Tries to change the SpeedStep state. Can be called from
+ * smp_call_function_single.
+ */
+static void speedstep_set_state(unsigned int state)
+{
+ u8 pm2_blk;
+ u8 value;
+ unsigned long flags;
+
+ if (state > 0x1)
+ return;
+
+ /* Disable IRQs */
+ local_irq_save(flags);
+
+ /* read state */
+ value = inb(pmbase + 0x50);
+
+ pr_debug("read at pmbase 0x%x + 0x50 returned 0x%x\n", pmbase, value);
+
+ /* write new state */
+ value &= 0xFE;
+ value |= state;
+
+ pr_debug("writing 0x%x to pmbase 0x%x + 0x50\n", value, pmbase);
+
+ /* Disable bus master arbitration */
+ pm2_blk = inb(pmbase + 0x20);
+ pm2_blk |= 0x01;
+ outb(pm2_blk, (pmbase + 0x20));
+
+ /* Actual transition */
+ outb(value, (pmbase + 0x50));
+
+ /* Restore bus master arbitration */
+ pm2_blk &= 0xfe;
+ outb(pm2_blk, (pmbase + 0x20));
+
+ /* check if transition was successful */
+ value = inb(pmbase + 0x50);
+
+ /* Enable IRQs */
+ local_irq_restore(flags);
+
+ pr_debug("read at pmbase 0x%x + 0x50 returned 0x%x\n", pmbase, value);
+
+ if (state == (value & 0x1))
+ pr_debug("change to %u MHz succeeded\n",
+ speedstep_get_frequency(speedstep_processor) / 1000);
+ else
+ pr_err("change failed - I/O error\n");
+
+ return;
+}
+
+/* Wrapper for smp_call_function_single. */
+static void _speedstep_set_state(void *_state)
+{
+ speedstep_set_state(*(unsigned int *)_state);
+}
+
+/**
+ * speedstep_activate - activate SpeedStep control in the chipset
+ *
+ * Tries to activate the SpeedStep status and control registers.
+ * Returns -EINVAL on an unsupported chipset, and zero on success.
+ */
+static int speedstep_activate(void)
+{
+ u16 value = 0;
+
+ if (!speedstep_chipset_dev)
+ return -EINVAL;
+
+ pci_read_config_word(speedstep_chipset_dev, 0x00A0, &value);
+ if (!(value & 0x08)) {
+ value |= 0x08;
+ pr_debug("activating SpeedStep (TM) registers\n");
+ pci_write_config_word(speedstep_chipset_dev, 0x00A0, value);
+ }
+
+ return 0;
+}
+
+
+/**
+ * speedstep_detect_chipset - detect the Southbridge which contains SpeedStep logic
+ *
+ * Detects ICH2-M, ICH3-M and ICH4-M so far. The pci_dev points to
+ * the LPC bridge / PM module which contains all power-management
+ * functions. Returns the SPEEDSTEP_CHIPSET_-number for the detected
+ * chipset, or zero on failure.
+ */
+static unsigned int speedstep_detect_chipset(void)
+{
+ speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL,
+ PCI_DEVICE_ID_INTEL_82801DB_12,
+ PCI_ANY_ID, PCI_ANY_ID,
+ NULL);
+ if (speedstep_chipset_dev)
+ return 4; /* 4-M */
+
+ speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL,
+ PCI_DEVICE_ID_INTEL_82801CA_12,
+ PCI_ANY_ID, PCI_ANY_ID,
+ NULL);
+ if (speedstep_chipset_dev)
+ return 3; /* 3-M */
+
+
+ speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL,
+ PCI_DEVICE_ID_INTEL_82801BA_10,
+ PCI_ANY_ID, PCI_ANY_ID,
+ NULL);
+ if (speedstep_chipset_dev) {
+ /* speedstep.c causes lockups on Dell Inspirons 8000 and
+ * 8100 which use a pretty old revision of the 82815
+ * host bridge. Abort on these systems.
+ */
+ struct pci_dev *hostbridge;
+
+ hostbridge = pci_get_subsys(PCI_VENDOR_ID_INTEL,
+ PCI_DEVICE_ID_INTEL_82815_MC,
+ PCI_ANY_ID, PCI_ANY_ID,
+ NULL);
+
+ if (!hostbridge)
+ return 2; /* 2-M */
+
+ if (hostbridge->revision < 5) {
+ pr_debug("hostbridge does not support speedstep\n");
+ speedstep_chipset_dev = NULL;
+ pci_dev_put(hostbridge);
+ return 0;
+ }
+
+ pci_dev_put(hostbridge);
+ return 2; /* 2-M */
+ }
+
+ return 0;
+}
+
+static void get_freq_data(void *_speed)
+{
+ unsigned int *speed = _speed;
+
+ *speed = speedstep_get_frequency(speedstep_processor);
+}
+
+static unsigned int speedstep_get(unsigned int cpu)
+{
+ unsigned int speed;
+
+ /* You're supposed to ensure CPU is online. */
+ BUG_ON(smp_call_function_single(cpu, get_freq_data, &speed, 1));
+
+ pr_debug("detected %u kHz as current frequency\n", speed);
+ return speed;
+}
+
+/**
+ * speedstep_target - set a new CPUFreq policy
+ * @policy: new policy
+ * @index: index of target frequency
+ *
+ * Sets a new CPUFreq policy.
+ */
+static int speedstep_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ unsigned int policy_cpu;
+
+ policy_cpu = cpumask_any_and(policy->cpus, cpu_online_mask);
+
+ smp_call_function_single(policy_cpu, _speedstep_set_state, &index,
+ true);
+
+ return 0;
+}
+
+
+struct get_freqs {
+ struct cpufreq_policy *policy;
+ int ret;
+};
+
+static void get_freqs_on_cpu(void *_get_freqs)
+{
+ struct get_freqs *get_freqs = _get_freqs;
+
+ get_freqs->ret =
+ speedstep_get_freqs(speedstep_processor,
+ &speedstep_freqs[SPEEDSTEP_LOW].frequency,
+ &speedstep_freqs[SPEEDSTEP_HIGH].frequency,
+ &get_freqs->policy->cpuinfo.transition_latency,
+ &speedstep_set_state);
+}
+
+static int speedstep_cpu_init(struct cpufreq_policy *policy)
+{
+ unsigned int policy_cpu;
+ struct get_freqs gf;
+
+ /* only run on CPU to be set, or on its sibling */
+#ifdef CONFIG_SMP
+ cpumask_copy(policy->cpus, topology_sibling_cpumask(policy->cpu));
+#endif
+ policy_cpu = cpumask_any_and(policy->cpus, cpu_online_mask);
+
+ /* detect low and high frequency and transition latency */
+ gf.policy = policy;
+ smp_call_function_single(policy_cpu, get_freqs_on_cpu, &gf, 1);
+ if (gf.ret)
+ return gf.ret;
+
+ policy->freq_table = speedstep_freqs;
+
+ return 0;
+}
+
+
+static struct cpufreq_driver speedstep_driver = {
+ .name = "speedstep-ich",
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = speedstep_target,
+ .init = speedstep_cpu_init,
+ .get = speedstep_get,
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct x86_cpu_id ss_smi_ids[] = {
+ X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0x8, 0),
+ X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0xb, 0),
+ X86_MATCH_VENDOR_FAM_MODEL(INTEL, 15, 0x2, 0),
+ {}
+};
+
+/**
+ * speedstep_init - initializes the SpeedStep CPUFreq driver
+ *
+ * Initializes the SpeedStep support. Returns -ENODEV on unsupported
+ * devices, -EINVAL on problems during initiatization, and zero on
+ * success.
+ */
+static int __init speedstep_init(void)
+{
+ if (!x86_match_cpu(ss_smi_ids))
+ return -ENODEV;
+
+ /* detect processor */
+ speedstep_processor = speedstep_detect_processor();
+ if (!speedstep_processor) {
+ pr_debug("Intel(R) SpeedStep(TM) capable processor "
+ "not found\n");
+ return -ENODEV;
+ }
+
+ /* detect chipset */
+ if (!speedstep_detect_chipset()) {
+ pr_debug("Intel(R) SpeedStep(TM) for this chipset not "
+ "(yet) available.\n");
+ return -ENODEV;
+ }
+
+ /* activate speedstep support */
+ if (speedstep_activate()) {
+ pci_dev_put(speedstep_chipset_dev);
+ return -EINVAL;
+ }
+
+ if (speedstep_find_register())
+ return -ENODEV;
+
+ return cpufreq_register_driver(&speedstep_driver);
+}
+
+
+/**
+ * speedstep_exit - unregisters SpeedStep support
+ *
+ * Unregisters SpeedStep support.
+ */
+static void __exit speedstep_exit(void)
+{
+ pci_dev_put(speedstep_chipset_dev);
+ cpufreq_unregister_driver(&speedstep_driver);
+}
+
+
+MODULE_AUTHOR("Dave Jones, Dominik Brodowski <linux@brodo.de>");
+MODULE_DESCRIPTION("Speedstep driver for Intel mobile processors on chipsets "
+ "with ICH-M southbridges.");
+MODULE_LICENSE("GPL");
+
+module_init(speedstep_init);
+module_exit(speedstep_exit);
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
new file mode 100644
index 0000000000..0b66df4ed5
--- /dev/null
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -0,0 +1,479 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ *
+ * Library for common functions for Intel SpeedStep v.1 and v.2 support
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+
+#include <asm/msr.h>
+#include <asm/tsc.h>
+#include "speedstep-lib.h"
+
+#define PFX "speedstep-lib: "
+
+#ifdef CONFIG_X86_SPEEDSTEP_RELAXED_CAP_CHECK
+static int relaxed_check;
+#else
+#define relaxed_check 0
+#endif
+
+/*********************************************************************
+ * GET PROCESSOR CORE SPEED IN KHZ *
+ *********************************************************************/
+
+static unsigned int pentium3_get_frequency(enum speedstep_processor processor)
+{
+ /* See table 14 of p3_ds.pdf and table 22 of 29834003.pdf */
+ static const struct {
+ unsigned int ratio; /* Frequency Multiplier (x10) */
+ u8 bitmap; /* power on configuration bits
+ [27, 25:22] (in MSR 0x2a) */
+ } msr_decode_mult[] = {
+ { 30, 0x01 },
+ { 35, 0x05 },
+ { 40, 0x02 },
+ { 45, 0x06 },
+ { 50, 0x00 },
+ { 55, 0x04 },
+ { 60, 0x0b },
+ { 65, 0x0f },
+ { 70, 0x09 },
+ { 75, 0x0d },
+ { 80, 0x0a },
+ { 85, 0x26 },
+ { 90, 0x20 },
+ { 100, 0x2b },
+ { 0, 0xff } /* error or unknown value */
+ };
+
+ /* PIII(-M) FSB settings: see table b1-b of 24547206.pdf */
+ static const struct {
+ unsigned int value; /* Front Side Bus speed in MHz */
+ u8 bitmap; /* power on configuration bits [18: 19]
+ (in MSR 0x2a) */
+ } msr_decode_fsb[] = {
+ { 66, 0x0 },
+ { 100, 0x2 },
+ { 133, 0x1 },
+ { 0, 0xff}
+ };
+
+ u32 msr_lo, msr_tmp;
+ int i = 0, j = 0;
+
+ /* read MSR 0x2a - we only need the low 32 bits */
+ rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp);
+ pr_debug("P3 - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", msr_lo, msr_tmp);
+ msr_tmp = msr_lo;
+
+ /* decode the FSB */
+ msr_tmp &= 0x00c0000;
+ msr_tmp >>= 18;
+ while (msr_tmp != msr_decode_fsb[i].bitmap) {
+ if (msr_decode_fsb[i].bitmap == 0xff)
+ return 0;
+ i++;
+ }
+
+ /* decode the multiplier */
+ if (processor == SPEEDSTEP_CPU_PIII_C_EARLY) {
+ pr_debug("workaround for early PIIIs\n");
+ msr_lo &= 0x03c00000;
+ } else
+ msr_lo &= 0x0bc00000;
+ msr_lo >>= 22;
+ while (msr_lo != msr_decode_mult[j].bitmap) {
+ if (msr_decode_mult[j].bitmap == 0xff)
+ return 0;
+ j++;
+ }
+
+ pr_debug("speed is %u\n",
+ (msr_decode_mult[j].ratio * msr_decode_fsb[i].value * 100));
+
+ return msr_decode_mult[j].ratio * msr_decode_fsb[i].value * 100;
+}
+
+
+static unsigned int pentiumM_get_frequency(void)
+{
+ u32 msr_lo, msr_tmp;
+
+ rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp);
+ pr_debug("PM - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", msr_lo, msr_tmp);
+
+ /* see table B-2 of 24547212.pdf */
+ if (msr_lo & 0x00040000) {
+ printk(KERN_DEBUG PFX "PM - invalid FSB: 0x%x 0x%x\n",
+ msr_lo, msr_tmp);
+ return 0;
+ }
+
+ msr_tmp = (msr_lo >> 22) & 0x1f;
+ pr_debug("bits 22-26 are 0x%x, speed is %u\n",
+ msr_tmp, (msr_tmp * 100 * 1000));
+
+ return msr_tmp * 100 * 1000;
+}
+
+static unsigned int pentium_core_get_frequency(void)
+{
+ u32 fsb = 0;
+ u32 msr_lo, msr_tmp;
+ int ret;
+
+ rdmsr(MSR_FSB_FREQ, msr_lo, msr_tmp);
+ /* see table B-2 of 25366920.pdf */
+ switch (msr_lo & 0x07) {
+ case 5:
+ fsb = 100000;
+ break;
+ case 1:
+ fsb = 133333;
+ break;
+ case 3:
+ fsb = 166667;
+ break;
+ case 2:
+ fsb = 200000;
+ break;
+ case 0:
+ fsb = 266667;
+ break;
+ case 4:
+ fsb = 333333;
+ break;
+ default:
+ pr_err("PCORE - MSR_FSB_FREQ undefined value\n");
+ }
+
+ rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp);
+ pr_debug("PCORE - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n",
+ msr_lo, msr_tmp);
+
+ msr_tmp = (msr_lo >> 22) & 0x1f;
+ pr_debug("bits 22-26 are 0x%x, speed is %u\n",
+ msr_tmp, (msr_tmp * fsb));
+
+ ret = (msr_tmp * fsb);
+ return ret;
+}
+
+
+static unsigned int pentium4_get_frequency(void)
+{
+ struct cpuinfo_x86 *c = &boot_cpu_data;
+ u32 msr_lo, msr_hi, mult;
+ unsigned int fsb = 0;
+ unsigned int ret;
+ u8 fsb_code;
+
+ /* Pentium 4 Model 0 and 1 do not have the Core Clock Frequency
+ * to System Bus Frequency Ratio Field in the Processor Frequency
+ * Configuration Register of the MSR. Therefore the current
+ * frequency cannot be calculated and has to be measured.
+ */
+ if (c->x86_model < 2)
+ return cpu_khz;
+
+ rdmsr(0x2c, msr_lo, msr_hi);
+
+ pr_debug("P4 - MSR_EBC_FREQUENCY_ID: 0x%x 0x%x\n", msr_lo, msr_hi);
+
+ /* decode the FSB: see IA-32 Intel (C) Architecture Software
+ * Developer's Manual, Volume 3: System Prgramming Guide,
+ * revision #12 in Table B-1: MSRs in the Pentium 4 and
+ * Intel Xeon Processors, on page B-4 and B-5.
+ */
+ fsb_code = (msr_lo >> 16) & 0x7;
+ switch (fsb_code) {
+ case 0:
+ fsb = 100 * 1000;
+ break;
+ case 1:
+ fsb = 13333 * 10;
+ break;
+ case 2:
+ fsb = 200 * 1000;
+ break;
+ }
+
+ if (!fsb)
+ printk(KERN_DEBUG PFX "couldn't detect FSB speed. "
+ "Please send an e-mail to <linux@brodo.de>\n");
+
+ /* Multiplier. */
+ mult = msr_lo >> 24;
+
+ pr_debug("P4 - FSB %u kHz; Multiplier %u; Speed %u kHz\n",
+ fsb, mult, (fsb * mult));
+
+ ret = (fsb * mult);
+ return ret;
+}
+
+
+/* Warning: may get called from smp_call_function_single. */
+unsigned int speedstep_get_frequency(enum speedstep_processor processor)
+{
+ switch (processor) {
+ case SPEEDSTEP_CPU_PCORE:
+ return pentium_core_get_frequency();
+ case SPEEDSTEP_CPU_PM:
+ return pentiumM_get_frequency();
+ case SPEEDSTEP_CPU_P4D:
+ case SPEEDSTEP_CPU_P4M:
+ return pentium4_get_frequency();
+ case SPEEDSTEP_CPU_PIII_T:
+ case SPEEDSTEP_CPU_PIII_C:
+ case SPEEDSTEP_CPU_PIII_C_EARLY:
+ return pentium3_get_frequency(processor);
+ default:
+ return 0;
+ }
+ return 0;
+}
+EXPORT_SYMBOL_GPL(speedstep_get_frequency);
+
+
+/*********************************************************************
+ * DETECT SPEEDSTEP-CAPABLE PROCESSOR *
+ *********************************************************************/
+
+/* Keep in sync with the x86_cpu_id tables in the different modules */
+enum speedstep_processor speedstep_detect_processor(void)
+{
+ struct cpuinfo_x86 *c = &cpu_data(0);
+ u32 ebx, msr_lo, msr_hi;
+
+ pr_debug("x86: %x, model: %x\n", c->x86, c->x86_model);
+
+ if ((c->x86_vendor != X86_VENDOR_INTEL) ||
+ ((c->x86 != 6) && (c->x86 != 0xF)))
+ return 0;
+
+ if (c->x86 == 0xF) {
+ /* Intel Mobile Pentium 4-M
+ * or Intel Mobile Pentium 4 with 533 MHz FSB */
+ if (c->x86_model != 2)
+ return 0;
+
+ ebx = cpuid_ebx(0x00000001);
+ ebx &= 0x000000FF;
+
+ pr_debug("ebx value is %x, x86_stepping is %x\n", ebx, c->x86_stepping);
+
+ switch (c->x86_stepping) {
+ case 4:
+ /*
+ * B-stepping [M-P4-M]
+ * sample has ebx = 0x0f, production has 0x0e.
+ */
+ if ((ebx == 0x0e) || (ebx == 0x0f))
+ return SPEEDSTEP_CPU_P4M;
+ break;
+ case 7:
+ /*
+ * C-stepping [M-P4-M]
+ * needs to have ebx=0x0e, else it's a celeron:
+ * cf. 25130917.pdf / page 7, footnote 5 even
+ * though 25072120.pdf / page 7 doesn't say
+ * samples are only of B-stepping...
+ */
+ if (ebx == 0x0e)
+ return SPEEDSTEP_CPU_P4M;
+ break;
+ case 9:
+ /*
+ * D-stepping [M-P4-M or M-P4/533]
+ *
+ * this is totally strange: CPUID 0x0F29 is
+ * used by M-P4-M, M-P4/533 and(!) Celeron CPUs.
+ * The latter need to be sorted out as they don't
+ * support speedstep.
+ * Celerons with CPUID 0x0F29 may have either
+ * ebx=0x8 or 0xf -- 25130917.pdf doesn't say anything
+ * specific.
+ * M-P4-Ms may have either ebx=0xe or 0xf [see above]
+ * M-P4/533 have either ebx=0xe or 0xf. [25317607.pdf]
+ * also, M-P4M HTs have ebx=0x8, too
+ * For now, they are distinguished by the model_id
+ * string
+ */
+ if ((ebx == 0x0e) ||
+ (strstr(c->x86_model_id,
+ "Mobile Intel(R) Pentium(R) 4") != NULL))
+ return SPEEDSTEP_CPU_P4M;
+ break;
+ default:
+ break;
+ }
+ return 0;
+ }
+
+ switch (c->x86_model) {
+ case 0x0B: /* Intel PIII [Tualatin] */
+ /* cpuid_ebx(1) is 0x04 for desktop PIII,
+ * 0x06 for mobile PIII-M */
+ ebx = cpuid_ebx(0x00000001);
+ pr_debug("ebx is %x\n", ebx);
+
+ ebx &= 0x000000FF;
+
+ if (ebx != 0x06)
+ return 0;
+
+ /* So far all PIII-M processors support SpeedStep. See
+ * Intel's 24540640.pdf of June 2003
+ */
+ return SPEEDSTEP_CPU_PIII_T;
+
+ case 0x08: /* Intel PIII [Coppermine] */
+
+ /* all mobile PIII Coppermines have FSB 100 MHz
+ * ==> sort out a few desktop PIIIs. */
+ rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_hi);
+ pr_debug("Coppermine: MSR_IA32_EBL_CR_POWERON is 0x%x, 0x%x\n",
+ msr_lo, msr_hi);
+ msr_lo &= 0x00c0000;
+ if (msr_lo != 0x0080000)
+ return 0;
+
+ /*
+ * If the processor is a mobile version,
+ * platform ID has bit 50 set
+ * it has SpeedStep technology if either
+ * bit 56 or 57 is set
+ */
+ rdmsr(MSR_IA32_PLATFORM_ID, msr_lo, msr_hi);
+ pr_debug("Coppermine: MSR_IA32_PLATFORM ID is 0x%x, 0x%x\n",
+ msr_lo, msr_hi);
+ if ((msr_hi & (1<<18)) &&
+ (relaxed_check ? 1 : (msr_hi & (3<<24)))) {
+ if (c->x86_stepping == 0x01) {
+ pr_debug("early PIII version\n");
+ return SPEEDSTEP_CPU_PIII_C_EARLY;
+ } else
+ return SPEEDSTEP_CPU_PIII_C;
+ }
+ fallthrough;
+ default:
+ return 0;
+ }
+}
+EXPORT_SYMBOL_GPL(speedstep_detect_processor);
+
+
+/*********************************************************************
+ * DETECT SPEEDSTEP SPEEDS *
+ *********************************************************************/
+
+unsigned int speedstep_get_freqs(enum speedstep_processor processor,
+ unsigned int *low_speed,
+ unsigned int *high_speed,
+ unsigned int *transition_latency,
+ void (*set_state) (unsigned int state))
+{
+ unsigned int prev_speed;
+ unsigned int ret = 0;
+ unsigned long flags;
+ ktime_t tv1, tv2;
+
+ if ((!processor) || (!low_speed) || (!high_speed) || (!set_state))
+ return -EINVAL;
+
+ pr_debug("trying to determine both speeds\n");
+
+ /* get current speed */
+ prev_speed = speedstep_get_frequency(processor);
+ if (!prev_speed)
+ return -EIO;
+
+ pr_debug("previous speed is %u\n", prev_speed);
+
+ preempt_disable();
+ local_irq_save(flags);
+
+ /* switch to low state */
+ set_state(SPEEDSTEP_LOW);
+ *low_speed = speedstep_get_frequency(processor);
+ if (!*low_speed) {
+ ret = -EIO;
+ goto out;
+ }
+
+ pr_debug("low speed is %u\n", *low_speed);
+
+ /* start latency measurement */
+ if (transition_latency)
+ tv1 = ktime_get();
+
+ /* switch to high state */
+ set_state(SPEEDSTEP_HIGH);
+
+ /* end latency measurement */
+ if (transition_latency)
+ tv2 = ktime_get();
+
+ *high_speed = speedstep_get_frequency(processor);
+ if (!*high_speed) {
+ ret = -EIO;
+ goto out;
+ }
+
+ pr_debug("high speed is %u\n", *high_speed);
+
+ if (*low_speed == *high_speed) {
+ ret = -ENODEV;
+ goto out;
+ }
+
+ /* switch to previous state, if necessary */
+ if (*high_speed != prev_speed)
+ set_state(SPEEDSTEP_LOW);
+
+ if (transition_latency) {
+ *transition_latency = ktime_to_us(ktime_sub(tv2, tv1));
+ pr_debug("transition latency is %u uSec\n", *transition_latency);
+
+ /* convert uSec to nSec and add 20% for safety reasons */
+ *transition_latency *= 1200;
+
+ /* check if the latency measurement is too high or too low
+ * and set it to a safe value (500uSec) in that case
+ */
+ if (*transition_latency > 10000000 ||
+ *transition_latency < 50000) {
+ pr_warn("frequency transition measured seems out of range (%u nSec), falling back to a safe one of %u nSec\n",
+ *transition_latency, 500000);
+ *transition_latency = 500000;
+ }
+ }
+
+out:
+ local_irq_restore(flags);
+ preempt_enable();
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(speedstep_get_freqs);
+
+#ifdef CONFIG_X86_SPEEDSTEP_RELAXED_CAP_CHECK
+module_param(relaxed_check, int, 0444);
+MODULE_PARM_DESC(relaxed_check,
+ "Don't do all checks for speedstep capability.");
+#endif
+
+MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>");
+MODULE_DESCRIPTION("Library for Intel SpeedStep 1 or 2 cpufreq drivers.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/speedstep-lib.h b/drivers/cpufreq/speedstep-lib.h
new file mode 100644
index 0000000000..dc762ea786
--- /dev/null
+++ b/drivers/cpufreq/speedstep-lib.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
+ *
+ * Library for common functions for Intel SpeedStep v.1 and v.2 support
+ *
+ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous*
+ */
+
+
+
+/* processors */
+enum speedstep_processor {
+ SPEEDSTEP_CPU_PIII_C_EARLY = 0x00000001, /* Coppermine core */
+ SPEEDSTEP_CPU_PIII_C = 0x00000002, /* Coppermine core */
+ SPEEDSTEP_CPU_PIII_T = 0x00000003, /* Tualatin core */
+ SPEEDSTEP_CPU_P4M = 0x00000004, /* P4-M */
+/* the following processors are not speedstep-capable and are not auto-detected
+ * in speedstep_detect_processor(). However, their speed can be detected using
+ * the speedstep_get_frequency() call. */
+ SPEEDSTEP_CPU_PM = 0xFFFFFF03, /* Pentium M */
+ SPEEDSTEP_CPU_P4D = 0xFFFFFF04, /* desktop P4 */
+ SPEEDSTEP_CPU_PCORE = 0xFFFFFF05, /* Core */
+};
+
+/* speedstep states -- only two of them */
+
+#define SPEEDSTEP_HIGH 0x00000000
+#define SPEEDSTEP_LOW 0x00000001
+
+
+/* detect a speedstep-capable processor */
+extern enum speedstep_processor speedstep_detect_processor(void);
+
+/* detect the current speed (in khz) of the processor */
+extern unsigned int speedstep_get_frequency(enum speedstep_processor processor);
+
+
+/* detect the low and high speeds of the processor. The callback
+ * set_state"'s first argument is either SPEEDSTEP_HIGH or
+ * SPEEDSTEP_LOW; the second argument is zero so that no
+ * cpufreq_notify_transition calls are initiated.
+ */
+extern unsigned int speedstep_get_freqs(enum speedstep_processor processor,
+ unsigned int *low_speed,
+ unsigned int *high_speed,
+ unsigned int *transition_latency,
+ void (*set_state) (unsigned int state));
diff --git a/drivers/cpufreq/speedstep-smi.c b/drivers/cpufreq/speedstep-smi.c
new file mode 100644
index 0000000000..0ce9d4b6df
--- /dev/null
+++ b/drivers/cpufreq/speedstep-smi.c
@@ -0,0 +1,393 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel SpeedStep SMI driver.
+ *
+ * (C) 2003 Hiroshi Miura <miura@da-cha.org>
+ */
+
+
+/*********************************************************************
+ * SPEEDSTEP - DEFINITIONS *
+ *********************************************************************/
+
+#define pr_fmt(fmt) "cpufreq: " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/cpufreq.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <asm/ist.h>
+#include <asm/cpu_device_id.h>
+
+#include "speedstep-lib.h"
+
+/* speedstep system management interface port/command.
+ *
+ * These parameters are got from IST-SMI BIOS call.
+ * If user gives it, these are used.
+ *
+ */
+static int smi_port;
+static int smi_cmd;
+static unsigned int smi_sig;
+
+/* info about the processor */
+static enum speedstep_processor speedstep_processor;
+
+/*
+ * There are only two frequency states for each processor. Values
+ * are in kHz for the time being.
+ */
+static struct cpufreq_frequency_table speedstep_freqs[] = {
+ {0, SPEEDSTEP_HIGH, 0},
+ {0, SPEEDSTEP_LOW, 0},
+ {0, 0, CPUFREQ_TABLE_END},
+};
+
+#define GET_SPEEDSTEP_OWNER 0
+#define GET_SPEEDSTEP_STATE 1
+#define SET_SPEEDSTEP_STATE 2
+#define GET_SPEEDSTEP_FREQS 4
+
+/* how often shall the SMI call be tried if it failed, e.g. because
+ * of DMA activity going on? */
+#define SMI_TRIES 5
+
+/**
+ * speedstep_smi_ownership
+ */
+static int speedstep_smi_ownership(void)
+{
+ u32 command, result, magic, dummy;
+ u32 function = GET_SPEEDSTEP_OWNER;
+ unsigned char magic_data[] = "Copyright (c) 1999 Intel Corporation";
+
+ command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff);
+ magic = virt_to_phys(magic_data);
+
+ pr_debug("trying to obtain ownership with command %x at port %x\n",
+ command, smi_port);
+
+ __asm__ __volatile__(
+ "push %%ebp\n"
+ "out %%al, (%%dx)\n"
+ "pop %%ebp\n"
+ : "=D" (result),
+ "=a" (dummy), "=b" (dummy), "=c" (dummy), "=d" (dummy),
+ "=S" (dummy)
+ : "a" (command), "b" (function), "c" (0), "d" (smi_port),
+ "D" (0), "S" (magic)
+ : "memory"
+ );
+
+ pr_debug("result is %x\n", result);
+
+ return result;
+}
+
+/**
+ * speedstep_smi_get_freqs - get SpeedStep preferred & current freq.
+ * @low: the low frequency value is placed here
+ * @high: the high frequency value is placed here
+ *
+ * Only available on later SpeedStep-enabled systems, returns false results or
+ * even hangs [cf. bugme.osdl.org # 1422] on earlier systems. Empirical testing
+ * shows that the latter occurs if !(ist_info.event & 0xFFFF).
+ */
+static int speedstep_smi_get_freqs(unsigned int *low, unsigned int *high)
+{
+ u32 command, result = 0, edi, high_mhz, low_mhz, dummy;
+ u32 state = 0;
+ u32 function = GET_SPEEDSTEP_FREQS;
+
+ if (!(ist_info.event & 0xFFFF)) {
+ pr_debug("bug #1422 -- can't read freqs from BIOS\n");
+ return -ENODEV;
+ }
+
+ command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff);
+
+ pr_debug("trying to determine frequencies with command %x at port %x\n",
+ command, smi_port);
+
+ __asm__ __volatile__(
+ "push %%ebp\n"
+ "out %%al, (%%dx)\n"
+ "pop %%ebp"
+ : "=a" (result),
+ "=b" (high_mhz),
+ "=c" (low_mhz),
+ "=d" (state), "=D" (edi), "=S" (dummy)
+ : "a" (command),
+ "b" (function),
+ "c" (state),
+ "d" (smi_port), "S" (0), "D" (0)
+ );
+
+ pr_debug("result %x, low_freq %u, high_freq %u\n",
+ result, low_mhz, high_mhz);
+
+ /* abort if results are obviously incorrect... */
+ if ((high_mhz + low_mhz) < 600)
+ return -EINVAL;
+
+ *high = high_mhz * 1000;
+ *low = low_mhz * 1000;
+
+ return result;
+}
+
+/**
+ * speedstep_set_state - set the SpeedStep state
+ * @state: new processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH)
+ *
+ */
+static void speedstep_set_state(unsigned int state)
+{
+ unsigned int result = 0, command, new_state, dummy;
+ unsigned long flags;
+ unsigned int function = SET_SPEEDSTEP_STATE;
+ unsigned int retry = 0;
+
+ if (state > 0x1)
+ return;
+
+ /* Disable IRQs */
+ preempt_disable();
+ local_irq_save(flags);
+
+ command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff);
+
+ pr_debug("trying to set frequency to state %u "
+ "with command %x at port %x\n",
+ state, command, smi_port);
+
+ do {
+ if (retry) {
+ /*
+ * We need to enable interrupts, otherwise the blockage
+ * won't resolve.
+ *
+ * We disable preemption so that other processes don't
+ * run. If other processes were running, they could
+ * submit more DMA requests, making the blockage worse.
+ */
+ pr_debug("retry %u, previous result %u, waiting...\n",
+ retry, result);
+ local_irq_enable();
+ mdelay(retry * 50);
+ local_irq_disable();
+ }
+ retry++;
+ __asm__ __volatile__(
+ "push %%ebp\n"
+ "out %%al, (%%dx)\n"
+ "pop %%ebp"
+ : "=b" (new_state), "=D" (result),
+ "=c" (dummy), "=a" (dummy),
+ "=d" (dummy), "=S" (dummy)
+ : "a" (command), "b" (function), "c" (state),
+ "d" (smi_port), "S" (0), "D" (0)
+ );
+ } while ((new_state != state) && (retry <= SMI_TRIES));
+
+ /* enable IRQs */
+ local_irq_restore(flags);
+ preempt_enable();
+
+ if (new_state == state)
+ pr_debug("change to %u MHz succeeded after %u tries "
+ "with result %u\n",
+ (speedstep_freqs[new_state].frequency / 1000),
+ retry, result);
+ else
+ pr_err("change to state %u failed with new_state %u and result %u\n",
+ state, new_state, result);
+
+ return;
+}
+
+
+/**
+ * speedstep_target - set a new CPUFreq policy
+ * @policy: new policy
+ * @index: index of new freq
+ *
+ * Sets a new CPUFreq policy/freq.
+ */
+static int speedstep_target(struct cpufreq_policy *policy, unsigned int index)
+{
+ speedstep_set_state(index);
+
+ return 0;
+}
+
+
+static int speedstep_cpu_init(struct cpufreq_policy *policy)
+{
+ int result;
+ unsigned int *low, *high;
+
+ /* capability check */
+ if (policy->cpu != 0)
+ return -ENODEV;
+
+ result = speedstep_smi_ownership();
+ if (result) {
+ pr_debug("fails in acquiring ownership of a SMI interface.\n");
+ return -EINVAL;
+ }
+
+ /* detect low and high frequency */
+ low = &speedstep_freqs[SPEEDSTEP_LOW].frequency;
+ high = &speedstep_freqs[SPEEDSTEP_HIGH].frequency;
+
+ result = speedstep_smi_get_freqs(low, high);
+ if (result) {
+ /* fall back to speedstep_lib.c dection mechanism:
+ * try both states out */
+ pr_debug("could not detect low and high frequencies "
+ "by SMI call.\n");
+ result = speedstep_get_freqs(speedstep_processor,
+ low, high,
+ NULL,
+ &speedstep_set_state);
+
+ if (result) {
+ pr_debug("could not detect two different speeds"
+ " -- aborting.\n");
+ return result;
+ } else
+ pr_debug("workaround worked.\n");
+ }
+
+ policy->freq_table = speedstep_freqs;
+
+ return 0;
+}
+
+static unsigned int speedstep_get(unsigned int cpu)
+{
+ if (cpu)
+ return -ENODEV;
+ return speedstep_get_frequency(speedstep_processor);
+}
+
+
+static int speedstep_resume(struct cpufreq_policy *policy)
+{
+ int result = speedstep_smi_ownership();
+
+ if (result)
+ pr_debug("fails in re-acquiring ownership of a SMI interface.\n");
+
+ return result;
+}
+
+static struct cpufreq_driver speedstep_driver = {
+ .name = "speedstep-smi",
+ .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = speedstep_target,
+ .init = speedstep_cpu_init,
+ .get = speedstep_get,
+ .resume = speedstep_resume,
+ .attr = cpufreq_generic_attr,
+};
+
+static const struct x86_cpu_id ss_smi_ids[] = {
+ X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0x8, 0),
+ X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0xb, 0),
+ X86_MATCH_VENDOR_FAM_MODEL(INTEL, 15, 0x2, 0),
+ {}
+};
+
+/**
+ * speedstep_init - initializes the SpeedStep CPUFreq driver
+ *
+ * Initializes the SpeedStep support. Returns -ENODEV on unsupported
+ * BIOS, -EINVAL on problems during initiatization, and zero on
+ * success.
+ */
+static int __init speedstep_init(void)
+{
+ if (!x86_match_cpu(ss_smi_ids))
+ return -ENODEV;
+
+ speedstep_processor = speedstep_detect_processor();
+
+ switch (speedstep_processor) {
+ case SPEEDSTEP_CPU_PIII_T:
+ case SPEEDSTEP_CPU_PIII_C:
+ case SPEEDSTEP_CPU_PIII_C_EARLY:
+ break;
+ default:
+ speedstep_processor = 0;
+ }
+
+ if (!speedstep_processor) {
+ pr_debug("No supported Intel CPU detected.\n");
+ return -ENODEV;
+ }
+
+ pr_debug("signature:0x%.8x, command:0x%.8x, "
+ "event:0x%.8x, perf_level:0x%.8x.\n",
+ ist_info.signature, ist_info.command,
+ ist_info.event, ist_info.perf_level);
+
+ /* Error if no IST-SMI BIOS or no PARM
+ sig= 'ISGE' aka 'Intel Speedstep Gate E' */
+ if ((ist_info.signature != 0x47534943) && (
+ (smi_port == 0) || (smi_cmd == 0)))
+ return -ENODEV;
+
+ if (smi_sig == 1)
+ smi_sig = 0x47534943;
+ else
+ smi_sig = ist_info.signature;
+
+ /* setup smi_port from MODLULE_PARM or BIOS */
+ if ((smi_port > 0xff) || (smi_port < 0))
+ return -EINVAL;
+ else if (smi_port == 0)
+ smi_port = ist_info.command & 0xff;
+
+ if ((smi_cmd > 0xff) || (smi_cmd < 0))
+ return -EINVAL;
+ else if (smi_cmd == 0)
+ smi_cmd = (ist_info.command >> 16) & 0xff;
+
+ return cpufreq_register_driver(&speedstep_driver);
+}
+
+
+/**
+ * speedstep_exit - unregisters SpeedStep support
+ *
+ * Unregisters SpeedStep support.
+ */
+static void __exit speedstep_exit(void)
+{
+ cpufreq_unregister_driver(&speedstep_driver);
+}
+
+module_param_hw(smi_port, int, ioport, 0444);
+module_param(smi_cmd, int, 0444);
+module_param(smi_sig, uint, 0444);
+
+MODULE_PARM_DESC(smi_port, "Override the BIOS-given IST port with this value "
+ "-- Intel's default setting is 0xb2");
+MODULE_PARM_DESC(smi_cmd, "Override the BIOS-given IST command with this value "
+ "-- Intel's default setting is 0x82");
+MODULE_PARM_DESC(smi_sig, "Set to 1 to fake the IST signature when using the "
+ "SMI interface.");
+
+MODULE_AUTHOR("Hiroshi Miura");
+MODULE_DESCRIPTION("Speedstep driver for IST applet SMI interface.");
+MODULE_LICENSE("GPL");
+
+module_init(speedstep_init);
+module_exit(speedstep_exit);
diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c
new file mode 100644
index 0000000000..9c542e723a
--- /dev/null
+++ b/drivers/cpufreq/sti-cpufreq.c
@@ -0,0 +1,303 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Match running platform with pre-defined OPP values for CPUFreq
+ *
+ * Author: Ajit Pal Singh <ajitpal.singh@st.com>
+ * Lee Jones <lee.jones@linaro.org>
+ *
+ * Copyright (C) 2015 STMicroelectronics (R&D) Limited
+ */
+
+#include <linux/cpu.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/regmap.h>
+
+#define VERSION_ELEMENTS 3
+#define MAX_PCODE_NAME_LEN 7
+
+#define VERSION_SHIFT 28
+#define HW_INFO_INDEX 1
+#define MAJOR_ID_INDEX 1
+#define MINOR_ID_INDEX 2
+
+/*
+ * Only match on "suitable for ALL versions" entries
+ *
+ * This will be used with the BIT() macro. It sets the
+ * top bit of a 32bit value and is equal to 0x80000000.
+ */
+#define DEFAULT_VERSION 31
+
+enum {
+ PCODE = 0,
+ SUBSTRATE,
+ DVFS_MAX_REGFIELDS,
+};
+
+/**
+ * struct sti_cpufreq_ddata - ST CPUFreq Driver Data
+ *
+ * @cpu: CPU's OF node
+ * @syscfg_eng: Engineering Syscon register map
+ * @syscfg: Syscon register map
+ */
+static struct sti_cpufreq_ddata {
+ struct device *cpu;
+ struct regmap *syscfg_eng;
+ struct regmap *syscfg;
+} ddata;
+
+static int sti_cpufreq_fetch_major(void) {
+ struct device_node *np = ddata.cpu->of_node;
+ struct device *dev = ddata.cpu;
+ unsigned int major_offset;
+ unsigned int socid;
+ int ret;
+
+ ret = of_property_read_u32_index(np, "st,syscfg",
+ MAJOR_ID_INDEX, &major_offset);
+ if (ret) {
+ dev_err(dev, "No major number offset provided in %pOF [%d]\n",
+ np, ret);
+ return ret;
+ }
+
+ ret = regmap_read(ddata.syscfg, major_offset, &socid);
+ if (ret) {
+ dev_err(dev, "Failed to read major number from syscon [%d]\n",
+ ret);
+ return ret;
+ }
+
+ return ((socid >> VERSION_SHIFT) & 0xf) + 1;
+}
+
+static int sti_cpufreq_fetch_minor(void)
+{
+ struct device *dev = ddata.cpu;
+ struct device_node *np = dev->of_node;
+ unsigned int minor_offset;
+ unsigned int minid;
+ int ret;
+
+ ret = of_property_read_u32_index(np, "st,syscfg-eng",
+ MINOR_ID_INDEX, &minor_offset);
+ if (ret) {
+ dev_err(dev,
+ "No minor number offset provided %pOF [%d]\n",
+ np, ret);
+ return ret;
+ }
+
+ ret = regmap_read(ddata.syscfg_eng, minor_offset, &minid);
+ if (ret) {
+ dev_err(dev,
+ "Failed to read the minor number from syscon [%d]\n",
+ ret);
+ return ret;
+ }
+
+ return minid & 0xf;
+}
+
+static int sti_cpufreq_fetch_regmap_field(const struct reg_field *reg_fields,
+ int hw_info_offset, int field)
+{
+ struct regmap_field *regmap_field;
+ struct reg_field reg_field = reg_fields[field];
+ struct device *dev = ddata.cpu;
+ unsigned int value;
+ int ret;
+
+ reg_field.reg = hw_info_offset;
+ regmap_field = devm_regmap_field_alloc(dev,
+ ddata.syscfg_eng,
+ reg_field);
+ if (IS_ERR(regmap_field)) {
+ dev_err(dev, "Failed to allocate reg field\n");
+ return PTR_ERR(regmap_field);
+ }
+
+ ret = regmap_field_read(regmap_field, &value);
+ if (ret) {
+ dev_err(dev, "Failed to read %s code\n",
+ field ? "SUBSTRATE" : "PCODE");
+ return ret;
+ }
+
+ return value;
+}
+
+static const struct reg_field sti_stih407_dvfs_regfields[DVFS_MAX_REGFIELDS] = {
+ [PCODE] = REG_FIELD(0, 16, 19),
+ [SUBSTRATE] = REG_FIELD(0, 0, 2),
+};
+
+static const struct reg_field *sti_cpufreq_match(void)
+{
+ if (of_machine_is_compatible("st,stih407") ||
+ of_machine_is_compatible("st,stih410") ||
+ of_machine_is_compatible("st,stih418"))
+ return sti_stih407_dvfs_regfields;
+
+ return NULL;
+}
+
+static int sti_cpufreq_set_opp_info(void)
+{
+ struct device *dev = ddata.cpu;
+ struct device_node *np = dev->of_node;
+ const struct reg_field *reg_fields;
+ unsigned int hw_info_offset;
+ unsigned int version[VERSION_ELEMENTS];
+ int pcode, substrate, major, minor;
+ int opp_token, ret;
+ char name[MAX_PCODE_NAME_LEN];
+ struct dev_pm_opp_config config = {
+ .supported_hw = version,
+ .supported_hw_count = ARRAY_SIZE(version),
+ .prop_name = name,
+ };
+
+ reg_fields = sti_cpufreq_match();
+ if (!reg_fields) {
+ dev_err(dev, "This SoC doesn't support voltage scaling\n");
+ return -ENODEV;
+ }
+
+ ret = of_property_read_u32_index(np, "st,syscfg-eng",
+ HW_INFO_INDEX, &hw_info_offset);
+ if (ret) {
+ dev_warn(dev, "Failed to read HW info offset from DT\n");
+ substrate = DEFAULT_VERSION;
+ pcode = 0;
+ goto use_defaults;
+ }
+
+ pcode = sti_cpufreq_fetch_regmap_field(reg_fields,
+ hw_info_offset,
+ PCODE);
+ if (pcode < 0) {
+ dev_warn(dev, "Failed to obtain process code\n");
+ /* Use default pcode */
+ pcode = 0;
+ }
+
+ substrate = sti_cpufreq_fetch_regmap_field(reg_fields,
+ hw_info_offset,
+ SUBSTRATE);
+ if (substrate) {
+ dev_warn(dev, "Failed to obtain substrate code\n");
+ /* Use default substrate */
+ substrate = DEFAULT_VERSION;
+ }
+
+use_defaults:
+ major = sti_cpufreq_fetch_major();
+ if (major < 0) {
+ dev_err(dev, "Failed to obtain major version\n");
+ /* Use default major number */
+ major = DEFAULT_VERSION;
+ }
+
+ minor = sti_cpufreq_fetch_minor();
+ if (minor < 0) {
+ dev_err(dev, "Failed to obtain minor version\n");
+ /* Use default minor number */
+ minor = DEFAULT_VERSION;
+ }
+
+ snprintf(name, MAX_PCODE_NAME_LEN, "pcode%d", pcode);
+
+ version[0] = BIT(major);
+ version[1] = BIT(minor);
+ version[2] = BIT(substrate);
+
+ opp_token = dev_pm_opp_set_config(dev, &config);
+ if (opp_token < 0) {
+ dev_err(dev, "Failed to set OPP config\n");
+ return opp_token;
+ }
+
+ dev_dbg(dev, "pcode: %d major: %d minor: %d substrate: %d\n",
+ pcode, major, minor, substrate);
+ dev_dbg(dev, "version[0]: %x version[1]: %x version[2]: %x\n",
+ version[0], version[1], version[2]);
+
+ return 0;
+}
+
+static int sti_cpufreq_fetch_syscon_registers(void)
+{
+ struct device *dev = ddata.cpu;
+ struct device_node *np = dev->of_node;
+
+ ddata.syscfg = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
+ if (IS_ERR(ddata.syscfg)) {
+ dev_err(dev, "\"st,syscfg\" not supplied\n");
+ return PTR_ERR(ddata.syscfg);
+ }
+
+ ddata.syscfg_eng = syscon_regmap_lookup_by_phandle(np, "st,syscfg-eng");
+ if (IS_ERR(ddata.syscfg_eng)) {
+ dev_err(dev, "\"st,syscfg-eng\" not supplied\n");
+ return PTR_ERR(ddata.syscfg_eng);
+ }
+
+ return 0;
+}
+
+static int __init sti_cpufreq_init(void)
+{
+ int ret;
+
+ if ((!of_machine_is_compatible("st,stih407")) &&
+ (!of_machine_is_compatible("st,stih410")) &&
+ (!of_machine_is_compatible("st,stih418")))
+ return -ENODEV;
+
+ ddata.cpu = get_cpu_device(0);
+ if (!ddata.cpu) {
+ dev_err(ddata.cpu, "Failed to get device for CPU0\n");
+ goto skip_voltage_scaling;
+ }
+
+ if (!of_get_property(ddata.cpu->of_node, "operating-points-v2", NULL)) {
+ dev_err(ddata.cpu, "OPP-v2 not supported\n");
+ goto skip_voltage_scaling;
+ }
+
+ ret = sti_cpufreq_fetch_syscon_registers();
+ if (ret)
+ goto skip_voltage_scaling;
+
+ ret = sti_cpufreq_set_opp_info();
+ if (!ret)
+ goto register_cpufreq_dt;
+
+skip_voltage_scaling:
+ dev_err(ddata.cpu, "Not doing voltage scaling\n");
+
+register_cpufreq_dt:
+ platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+
+ return 0;
+}
+module_init(sti_cpufreq_init);
+
+static const struct of_device_id __maybe_unused sti_cpufreq_of_match[] = {
+ { .compatible = "st,stih407" },
+ { .compatible = "st,stih410" },
+ { },
+};
+MODULE_DEVICE_TABLE(of, sti_cpufreq_of_match);
+
+MODULE_DESCRIPTION("STMicroelectronics CPUFreq/OPP driver");
+MODULE_AUTHOR("Ajitpal Singh <ajitpal.singh@st.com>");
+MODULE_AUTHOR("Lee Jones <lee.jones@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/sun50i-cpufreq-nvmem.c b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
new file mode 100644
index 0000000000..32a9c88f8f
--- /dev/null
+++ b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Allwinner CPUFreq nvmem based driver
+ *
+ * The sun50i-cpufreq-nvmem driver reads the efuse value from the SoC to
+ * provide the OPP framework with required information.
+ *
+ * Copyright (C) 2019 Yangtao Li <tiny.windzz@gmail.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cpu.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+
+#define MAX_NAME_LEN 7
+
+#define NVMEM_MASK 0x7
+#define NVMEM_SHIFT 5
+
+static struct platform_device *cpufreq_dt_pdev, *sun50i_cpufreq_pdev;
+
+/**
+ * sun50i_cpufreq_get_efuse() - Determine speed grade from efuse value
+ * @versions: Set to the value parsed from efuse
+ *
+ * Returns 0 if success.
+ */
+static int sun50i_cpufreq_get_efuse(u32 *versions)
+{
+ struct nvmem_cell *speedbin_nvmem;
+ struct device_node *np;
+ struct device *cpu_dev;
+ u32 *speedbin, efuse_value;
+ size_t len;
+ int ret;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev)
+ return -ENODEV;
+
+ np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
+ if (!np)
+ return -ENOENT;
+
+ ret = of_device_is_compatible(np,
+ "allwinner,sun50i-h6-operating-points");
+ if (!ret) {
+ of_node_put(np);
+ return -ENOENT;
+ }
+
+ speedbin_nvmem = of_nvmem_cell_get(np, NULL);
+ of_node_put(np);
+ if (IS_ERR(speedbin_nvmem))
+ return dev_err_probe(cpu_dev, PTR_ERR(speedbin_nvmem),
+ "Could not get nvmem cell\n");
+
+ speedbin = nvmem_cell_read(speedbin_nvmem, &len);
+ nvmem_cell_put(speedbin_nvmem);
+ if (IS_ERR(speedbin))
+ return PTR_ERR(speedbin);
+
+ efuse_value = (*speedbin >> NVMEM_SHIFT) & NVMEM_MASK;
+
+ /*
+ * We treat unexpected efuse values as if the SoC was from
+ * the slowest bin. Expected efuse values are 1-3, slowest
+ * to fastest.
+ */
+ if (efuse_value >= 1 && efuse_value <= 3)
+ *versions = efuse_value - 1;
+ else
+ *versions = 0;
+
+ kfree(speedbin);
+ return 0;
+};
+
+static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev)
+{
+ int *opp_tokens;
+ char name[MAX_NAME_LEN];
+ unsigned int cpu;
+ u32 speed = 0;
+ int ret;
+
+ opp_tokens = kcalloc(num_possible_cpus(), sizeof(*opp_tokens),
+ GFP_KERNEL);
+ if (!opp_tokens)
+ return -ENOMEM;
+
+ ret = sun50i_cpufreq_get_efuse(&speed);
+ if (ret) {
+ kfree(opp_tokens);
+ return ret;
+ }
+
+ snprintf(name, MAX_NAME_LEN, "speed%d", speed);
+
+ for_each_possible_cpu(cpu) {
+ struct device *cpu_dev = get_cpu_device(cpu);
+
+ if (!cpu_dev) {
+ ret = -ENODEV;
+ goto free_opp;
+ }
+
+ opp_tokens[cpu] = dev_pm_opp_set_prop_name(cpu_dev, name);
+ if (opp_tokens[cpu] < 0) {
+ ret = opp_tokens[cpu];
+ pr_err("Failed to set prop name\n");
+ goto free_opp;
+ }
+ }
+
+ cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1,
+ NULL, 0);
+ if (!IS_ERR(cpufreq_dt_pdev)) {
+ platform_set_drvdata(pdev, opp_tokens);
+ return 0;
+ }
+
+ ret = PTR_ERR(cpufreq_dt_pdev);
+ pr_err("Failed to register platform device\n");
+
+free_opp:
+ for_each_possible_cpu(cpu)
+ dev_pm_opp_put_prop_name(opp_tokens[cpu]);
+ kfree(opp_tokens);
+
+ return ret;
+}
+
+static void sun50i_cpufreq_nvmem_remove(struct platform_device *pdev)
+{
+ int *opp_tokens = platform_get_drvdata(pdev);
+ unsigned int cpu;
+
+ platform_device_unregister(cpufreq_dt_pdev);
+
+ for_each_possible_cpu(cpu)
+ dev_pm_opp_put_prop_name(opp_tokens[cpu]);
+
+ kfree(opp_tokens);
+}
+
+static struct platform_driver sun50i_cpufreq_driver = {
+ .probe = sun50i_cpufreq_nvmem_probe,
+ .remove_new = sun50i_cpufreq_nvmem_remove,
+ .driver = {
+ .name = "sun50i-cpufreq-nvmem",
+ },
+};
+
+static const struct of_device_id sun50i_cpufreq_match_list[] = {
+ { .compatible = "allwinner,sun50i-h6" },
+ {}
+};
+MODULE_DEVICE_TABLE(of, sun50i_cpufreq_match_list);
+
+static const struct of_device_id *sun50i_cpufreq_match_node(void)
+{
+ const struct of_device_id *match;
+ struct device_node *np;
+
+ np = of_find_node_by_path("/");
+ match = of_match_node(sun50i_cpufreq_match_list, np);
+ of_node_put(np);
+
+ return match;
+}
+
+/*
+ * Since the driver depends on nvmem drivers, which may return EPROBE_DEFER,
+ * all the real activity is done in the probe, which may be defered as well.
+ * The init here is only registering the driver and the platform device.
+ */
+static int __init sun50i_cpufreq_init(void)
+{
+ const struct of_device_id *match;
+ int ret;
+
+ match = sun50i_cpufreq_match_node();
+ if (!match)
+ return -ENODEV;
+
+ ret = platform_driver_register(&sun50i_cpufreq_driver);
+ if (unlikely(ret < 0))
+ return ret;
+
+ sun50i_cpufreq_pdev =
+ platform_device_register_simple("sun50i-cpufreq-nvmem",
+ -1, NULL, 0);
+ ret = PTR_ERR_OR_ZERO(sun50i_cpufreq_pdev);
+ if (ret == 0)
+ return 0;
+
+ platform_driver_unregister(&sun50i_cpufreq_driver);
+ return ret;
+}
+module_init(sun50i_cpufreq_init);
+
+static void __exit sun50i_cpufreq_exit(void)
+{
+ platform_device_unregister(sun50i_cpufreq_pdev);
+ platform_driver_unregister(&sun50i_cpufreq_driver);
+}
+module_exit(sun50i_cpufreq_exit);
+
+MODULE_DESCRIPTION("Sun50i-h6 cpufreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/tegra124-cpufreq.c b/drivers/cpufreq/tegra124-cpufreq.c
new file mode 100644
index 0000000000..aae951d4e7
--- /dev/null
+++ b/drivers/cpufreq/tegra124-cpufreq.c
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Tegra 124 cpufreq driver
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/types.h>
+
+struct tegra124_cpufreq_priv {
+ struct clk *cpu_clk;
+ struct clk *pllp_clk;
+ struct clk *pllx_clk;
+ struct clk *dfll_clk;
+ struct platform_device *cpufreq_dt_pdev;
+};
+
+static int tegra124_cpu_switch_to_dfll(struct tegra124_cpufreq_priv *priv)
+{
+ struct clk *orig_parent;
+ int ret;
+
+ ret = clk_set_rate(priv->dfll_clk, clk_get_rate(priv->cpu_clk));
+ if (ret)
+ return ret;
+
+ orig_parent = clk_get_parent(priv->cpu_clk);
+ clk_set_parent(priv->cpu_clk, priv->pllp_clk);
+
+ ret = clk_prepare_enable(priv->dfll_clk);
+ if (ret)
+ goto out;
+
+ clk_set_parent(priv->cpu_clk, priv->dfll_clk);
+
+ return 0;
+
+out:
+ clk_set_parent(priv->cpu_clk, orig_parent);
+
+ return ret;
+}
+
+static int tegra124_cpufreq_probe(struct platform_device *pdev)
+{
+ struct tegra124_cpufreq_priv *priv;
+ struct device_node *np;
+ struct device *cpu_dev;
+ struct platform_device_info cpufreq_dt_devinfo = {};
+ int ret;
+
+ priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev)
+ return -ENODEV;
+
+ np = of_cpu_device_node_get(0);
+ if (!np)
+ return -ENODEV;
+
+ priv->cpu_clk = of_clk_get_by_name(np, "cpu_g");
+ if (IS_ERR(priv->cpu_clk)) {
+ ret = PTR_ERR(priv->cpu_clk);
+ goto out_put_np;
+ }
+
+ priv->dfll_clk = of_clk_get_by_name(np, "dfll");
+ if (IS_ERR(priv->dfll_clk)) {
+ ret = PTR_ERR(priv->dfll_clk);
+ goto out_put_cpu_clk;
+ }
+
+ priv->pllx_clk = of_clk_get_by_name(np, "pll_x");
+ if (IS_ERR(priv->pllx_clk)) {
+ ret = PTR_ERR(priv->pllx_clk);
+ goto out_put_dfll_clk;
+ }
+
+ priv->pllp_clk = of_clk_get_by_name(np, "pll_p");
+ if (IS_ERR(priv->pllp_clk)) {
+ ret = PTR_ERR(priv->pllp_clk);
+ goto out_put_pllx_clk;
+ }
+
+ ret = tegra124_cpu_switch_to_dfll(priv);
+ if (ret)
+ goto out_put_pllp_clk;
+
+ cpufreq_dt_devinfo.name = "cpufreq-dt";
+ cpufreq_dt_devinfo.parent = &pdev->dev;
+
+ priv->cpufreq_dt_pdev =
+ platform_device_register_full(&cpufreq_dt_devinfo);
+ if (IS_ERR(priv->cpufreq_dt_pdev)) {
+ ret = PTR_ERR(priv->cpufreq_dt_pdev);
+ goto out_put_pllp_clk;
+ }
+
+ platform_set_drvdata(pdev, priv);
+
+ of_node_put(np);
+
+ return 0;
+
+out_put_pllp_clk:
+ clk_put(priv->pllp_clk);
+out_put_pllx_clk:
+ clk_put(priv->pllx_clk);
+out_put_dfll_clk:
+ clk_put(priv->dfll_clk);
+out_put_cpu_clk:
+ clk_put(priv->cpu_clk);
+out_put_np:
+ of_node_put(np);
+
+ return ret;
+}
+
+static int __maybe_unused tegra124_cpufreq_suspend(struct device *dev)
+{
+ struct tegra124_cpufreq_priv *priv = dev_get_drvdata(dev);
+ int err;
+
+ /*
+ * PLLP rate 408Mhz is below the CPU Fmax at Vmin and is safe to
+ * use during suspend and resume. So, switch the CPU clock source
+ * to PLLP and disable DFLL.
+ */
+ err = clk_set_parent(priv->cpu_clk, priv->pllp_clk);
+ if (err < 0) {
+ dev_err(dev, "failed to reparent to PLLP: %d\n", err);
+ return err;
+ }
+
+ clk_disable_unprepare(priv->dfll_clk);
+
+ return 0;
+}
+
+static int __maybe_unused tegra124_cpufreq_resume(struct device *dev)
+{
+ struct tegra124_cpufreq_priv *priv = dev_get_drvdata(dev);
+ int err;
+
+ /*
+ * Warmboot code powers up the CPU with PLLP clock source.
+ * Enable DFLL clock and switch CPU clock source back to DFLL.
+ */
+ err = clk_prepare_enable(priv->dfll_clk);
+ if (err < 0) {
+ dev_err(dev, "failed to enable DFLL clock for CPU: %d\n", err);
+ goto disable_cpufreq;
+ }
+
+ err = clk_set_parent(priv->cpu_clk, priv->dfll_clk);
+ if (err < 0) {
+ dev_err(dev, "failed to reparent to DFLL clock: %d\n", err);
+ goto disable_dfll;
+ }
+
+ return 0;
+
+disable_dfll:
+ clk_disable_unprepare(priv->dfll_clk);
+disable_cpufreq:
+ disable_cpufreq();
+
+ return err;
+}
+
+static const struct dev_pm_ops tegra124_cpufreq_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(tegra124_cpufreq_suspend,
+ tegra124_cpufreq_resume)
+};
+
+static struct platform_driver tegra124_cpufreq_platdrv = {
+ .driver.name = "cpufreq-tegra124",
+ .driver.pm = &tegra124_cpufreq_pm_ops,
+ .probe = tegra124_cpufreq_probe,
+};
+
+static int __init tegra_cpufreq_init(void)
+{
+ int ret;
+ struct platform_device *pdev;
+
+ if (!(of_machine_is_compatible("nvidia,tegra124") ||
+ of_machine_is_compatible("nvidia,tegra210")))
+ return -ENODEV;
+
+ /*
+ * Platform driver+device required for handling EPROBE_DEFER with
+ * the regulator and the DFLL clock
+ */
+ ret = platform_driver_register(&tegra124_cpufreq_platdrv);
+ if (ret)
+ return ret;
+
+ pdev = platform_device_register_simple("cpufreq-tegra124", -1, NULL, 0);
+ if (IS_ERR(pdev)) {
+ platform_driver_unregister(&tegra124_cpufreq_platdrv);
+ return PTR_ERR(pdev);
+ }
+
+ return 0;
+}
+module_init(tegra_cpufreq_init);
+
+MODULE_AUTHOR("Tuomas Tynkkynen <ttynkkynen@nvidia.com>");
+MODULE_DESCRIPTION("cpufreq driver for NVIDIA Tegra124");
diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c
new file mode 100644
index 0000000000..7b8fcfa550
--- /dev/null
+++ b/drivers/cpufreq/tegra186-cpufreq.c
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include <soc/tegra/bpmp.h>
+#include <soc/tegra/bpmp-abi.h>
+
+#define TEGRA186_NUM_CLUSTERS 2
+#define EDVD_OFFSET_A57(core) ((SZ_64K * 6) + (0x20 + (core) * 0x4))
+#define EDVD_OFFSET_DENVER(core) ((SZ_64K * 7) + (0x20 + (core) * 0x4))
+#define EDVD_CORE_VOLT_FREQ_F_SHIFT 0
+#define EDVD_CORE_VOLT_FREQ_F_MASK 0xffff
+#define EDVD_CORE_VOLT_FREQ_V_SHIFT 16
+
+struct tegra186_cpufreq_cpu {
+ unsigned int bpmp_cluster_id;
+ unsigned int edvd_offset;
+};
+
+static const struct tegra186_cpufreq_cpu tegra186_cpus[] = {
+ /* CPU0 - A57 Cluster */
+ {
+ .bpmp_cluster_id = 1,
+ .edvd_offset = EDVD_OFFSET_A57(0)
+ },
+ /* CPU1 - Denver Cluster */
+ {
+ .bpmp_cluster_id = 0,
+ .edvd_offset = EDVD_OFFSET_DENVER(0)
+ },
+ /* CPU2 - Denver Cluster */
+ {
+ .bpmp_cluster_id = 0,
+ .edvd_offset = EDVD_OFFSET_DENVER(1)
+ },
+ /* CPU3 - A57 Cluster */
+ {
+ .bpmp_cluster_id = 1,
+ .edvd_offset = EDVD_OFFSET_A57(1)
+ },
+ /* CPU4 - A57 Cluster */
+ {
+ .bpmp_cluster_id = 1,
+ .edvd_offset = EDVD_OFFSET_A57(2)
+ },
+ /* CPU5 - A57 Cluster */
+ {
+ .bpmp_cluster_id = 1,
+ .edvd_offset = EDVD_OFFSET_A57(3)
+ },
+};
+
+struct tegra186_cpufreq_cluster {
+ struct cpufreq_frequency_table *table;
+ u32 ref_clk_khz;
+ u32 div;
+};
+
+struct tegra186_cpufreq_data {
+ void __iomem *regs;
+ const struct tegra186_cpufreq_cpu *cpus;
+ struct tegra186_cpufreq_cluster clusters[];
+};
+
+static int tegra186_cpufreq_init(struct cpufreq_policy *policy)
+{
+ struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
+ unsigned int cluster = data->cpus[policy->cpu].bpmp_cluster_id;
+
+ policy->freq_table = data->clusters[cluster].table;
+ policy->cpuinfo.transition_latency = 300 * 1000;
+ policy->driver_data = NULL;
+
+ return 0;
+}
+
+static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
+ struct cpufreq_frequency_table *tbl = policy->freq_table + index;
+ unsigned int edvd_offset = data->cpus[policy->cpu].edvd_offset;
+ u32 edvd_val = tbl->driver_data;
+
+ writel(edvd_val, data->regs + edvd_offset);
+
+ return 0;
+}
+
+static unsigned int tegra186_cpufreq_get(unsigned int cpu)
+{
+ struct tegra186_cpufreq_data *data = cpufreq_get_driver_data();
+ struct tegra186_cpufreq_cluster *cluster;
+ struct cpufreq_policy *policy;
+ unsigned int edvd_offset, cluster_id;
+ u32 ndiv;
+
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy)
+ return 0;
+
+ edvd_offset = data->cpus[policy->cpu].edvd_offset;
+ ndiv = readl(data->regs + edvd_offset) & EDVD_CORE_VOLT_FREQ_F_MASK;
+ cluster_id = data->cpus[policy->cpu].bpmp_cluster_id;
+ cluster = &data->clusters[cluster_id];
+ cpufreq_cpu_put(policy);
+
+ return (cluster->ref_clk_khz * ndiv) / cluster->div;
+}
+
+static struct cpufreq_driver tegra186_cpufreq_driver = {
+ .name = "tegra186",
+ .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .get = tegra186_cpufreq_get,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = tegra186_cpufreq_set_target,
+ .init = tegra186_cpufreq_init,
+ .attr = cpufreq_generic_attr,
+};
+
+static struct cpufreq_frequency_table *init_vhint_table(
+ struct platform_device *pdev, struct tegra_bpmp *bpmp,
+ struct tegra186_cpufreq_cluster *cluster, unsigned int cluster_id)
+{
+ struct cpufreq_frequency_table *table;
+ struct mrq_cpu_vhint_request req;
+ struct tegra_bpmp_message msg;
+ struct cpu_vhint_data *data;
+ int err, i, j, num_rates = 0;
+ dma_addr_t phys;
+ void *virt;
+
+ virt = dma_alloc_coherent(bpmp->dev, sizeof(*data), &phys,
+ GFP_KERNEL);
+ if (!virt)
+ return ERR_PTR(-ENOMEM);
+
+ data = (struct cpu_vhint_data *)virt;
+
+ memset(&req, 0, sizeof(req));
+ req.addr = phys;
+ req.cluster_id = cluster_id;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.mrq = MRQ_CPU_VHINT;
+ msg.tx.data = &req;
+ msg.tx.size = sizeof(req);
+
+ err = tegra_bpmp_transfer(bpmp, &msg);
+ if (err) {
+ table = ERR_PTR(err);
+ goto free;
+ }
+ if (msg.rx.ret) {
+ table = ERR_PTR(-EINVAL);
+ goto free;
+ }
+
+ for (i = data->vfloor; i <= data->vceil; i++) {
+ u16 ndiv = data->ndiv[i];
+
+ if (ndiv < data->ndiv_min || ndiv > data->ndiv_max)
+ continue;
+
+ /* Only store lowest voltage index for each rate */
+ if (i > 0 && ndiv == data->ndiv[i - 1])
+ continue;
+
+ num_rates++;
+ }
+
+ table = devm_kcalloc(&pdev->dev, num_rates + 1, sizeof(*table),
+ GFP_KERNEL);
+ if (!table) {
+ table = ERR_PTR(-ENOMEM);
+ goto free;
+ }
+
+ cluster->ref_clk_khz = data->ref_clk_hz / 1000;
+ cluster->div = data->pdiv * data->mdiv;
+
+ for (i = data->vfloor, j = 0; i <= data->vceil; i++) {
+ struct cpufreq_frequency_table *point;
+ u16 ndiv = data->ndiv[i];
+ u32 edvd_val = 0;
+
+ if (ndiv < data->ndiv_min || ndiv > data->ndiv_max)
+ continue;
+
+ /* Only store lowest voltage index for each rate */
+ if (i > 0 && ndiv == data->ndiv[i - 1])
+ continue;
+
+ edvd_val |= i << EDVD_CORE_VOLT_FREQ_V_SHIFT;
+ edvd_val |= ndiv << EDVD_CORE_VOLT_FREQ_F_SHIFT;
+
+ point = &table[j++];
+ point->driver_data = edvd_val;
+ point->frequency = (cluster->ref_clk_khz * ndiv) / cluster->div;
+ }
+
+ table[j].frequency = CPUFREQ_TABLE_END;
+
+free:
+ dma_free_coherent(bpmp->dev, sizeof(*data), virt, phys);
+
+ return table;
+}
+
+static int tegra186_cpufreq_probe(struct platform_device *pdev)
+{
+ struct tegra186_cpufreq_data *data;
+ struct tegra_bpmp *bpmp;
+ unsigned int i = 0, err;
+
+ data = devm_kzalloc(&pdev->dev,
+ struct_size(data, clusters, TEGRA186_NUM_CLUSTERS),
+ GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ data->cpus = tegra186_cpus;
+
+ bpmp = tegra_bpmp_get(&pdev->dev);
+ if (IS_ERR(bpmp))
+ return PTR_ERR(bpmp);
+
+ data->regs = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(data->regs)) {
+ err = PTR_ERR(data->regs);
+ goto put_bpmp;
+ }
+
+ for (i = 0; i < TEGRA186_NUM_CLUSTERS; i++) {
+ struct tegra186_cpufreq_cluster *cluster = &data->clusters[i];
+
+ cluster->table = init_vhint_table(pdev, bpmp, cluster, i);
+ if (IS_ERR(cluster->table)) {
+ err = PTR_ERR(cluster->table);
+ goto put_bpmp;
+ }
+ }
+
+ tegra186_cpufreq_driver.driver_data = data;
+
+ err = cpufreq_register_driver(&tegra186_cpufreq_driver);
+
+put_bpmp:
+ tegra_bpmp_put(bpmp);
+
+ return err;
+}
+
+static void tegra186_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&tegra186_cpufreq_driver);
+}
+
+static const struct of_device_id tegra186_cpufreq_of_match[] = {
+ { .compatible = "nvidia,tegra186-ccplex-cluster", },
+ { }
+};
+MODULE_DEVICE_TABLE(of, tegra186_cpufreq_of_match);
+
+static struct platform_driver tegra186_cpufreq_platform_driver = {
+ .driver = {
+ .name = "tegra186-cpufreq",
+ .of_match_table = tegra186_cpufreq_of_match,
+ },
+ .probe = tegra186_cpufreq_probe,
+ .remove_new = tegra186_cpufreq_remove,
+};
+module_platform_driver(tegra186_cpufreq_platform_driver);
+
+MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra186 cpufreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c
new file mode 100644
index 0000000000..386aed3637
--- /dev/null
+++ b/drivers/cpufreq/tegra194-cpufreq.c
@@ -0,0 +1,769 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020 - 2022, NVIDIA CORPORATION. All rights reserved
+ */
+
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/units.h>
+
+#include <asm/smp_plat.h>
+
+#include <soc/tegra/bpmp.h>
+#include <soc/tegra/bpmp-abi.h>
+
+#define KHZ 1000
+#define REF_CLK_MHZ 408 /* 408 MHz */
+#define US_DELAY 500
+#define CPUFREQ_TBL_STEP_HZ (50 * KHZ * KHZ)
+#define MAX_CNT ~0U
+
+#define NDIV_MASK 0x1FF
+
+#define CORE_OFFSET(cpu) (cpu * 8)
+#define CMU_CLKS_BASE 0x2000
+#define SCRATCH_FREQ_CORE_REG(data, cpu) (data->regs + CMU_CLKS_BASE + CORE_OFFSET(cpu))
+
+#define MMCRAB_CLUSTER_BASE(cl) (0x30000 + (cl * 0x10000))
+#define CLUSTER_ACTMON_BASE(data, cl) \
+ (data->regs + (MMCRAB_CLUSTER_BASE(cl) + data->soc->actmon_cntr_base))
+#define CORE_ACTMON_CNTR_REG(data, cl, cpu) (CLUSTER_ACTMON_BASE(data, cl) + CORE_OFFSET(cpu))
+
+/* cpufreq transisition latency */
+#define TEGRA_CPUFREQ_TRANSITION_LATENCY (300 * 1000) /* unit in nanoseconds */
+
+struct tegra_cpu_ctr {
+ u32 cpu;
+ u32 coreclk_cnt, last_coreclk_cnt;
+ u32 refclk_cnt, last_refclk_cnt;
+};
+
+struct read_counters_work {
+ struct work_struct work;
+ struct tegra_cpu_ctr c;
+};
+
+struct tegra_cpufreq_ops {
+ void (*read_counters)(struct tegra_cpu_ctr *c);
+ void (*set_cpu_ndiv)(struct cpufreq_policy *policy, u64 ndiv);
+ void (*get_cpu_cluster_id)(u32 cpu, u32 *cpuid, u32 *clusterid);
+ int (*get_cpu_ndiv)(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv);
+};
+
+struct tegra_cpufreq_soc {
+ struct tegra_cpufreq_ops *ops;
+ int maxcpus_per_cluster;
+ unsigned int num_clusters;
+ phys_addr_t actmon_cntr_base;
+};
+
+struct tegra194_cpufreq_data {
+ void __iomem *regs;
+ struct cpufreq_frequency_table **bpmp_luts;
+ const struct tegra_cpufreq_soc *soc;
+ bool icc_dram_bw_scaling;
+};
+
+static struct workqueue_struct *read_counters_wq;
+
+static int tegra_cpufreq_set_bw(struct cpufreq_policy *policy, unsigned long freq_khz)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ struct dev_pm_opp *opp;
+ struct device *dev;
+ int ret;
+
+ dev = get_cpu_device(policy->cpu);
+ if (!dev)
+ return -ENODEV;
+
+ opp = dev_pm_opp_find_freq_exact(dev, freq_khz * KHZ, true);
+ if (IS_ERR(opp))
+ return PTR_ERR(opp);
+
+ ret = dev_pm_opp_set_opp(dev, opp);
+ if (ret)
+ data->icc_dram_bw_scaling = false;
+
+ dev_pm_opp_put(opp);
+ return ret;
+}
+
+static void tegra_get_cpu_mpidr(void *mpidr)
+{
+ *((u64 *)mpidr) = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
+}
+
+static void tegra234_get_cpu_cluster_id(u32 cpu, u32 *cpuid, u32 *clusterid)
+{
+ u64 mpidr;
+
+ smp_call_function_single(cpu, tegra_get_cpu_mpidr, &mpidr, true);
+
+ if (cpuid)
+ *cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+ if (clusterid)
+ *clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 2);
+}
+
+static int tegra234_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ void __iomem *freq_core_reg;
+ u64 mpidr_id;
+
+ /* use physical id to get address of per core frequency register */
+ mpidr_id = (clusterid * data->soc->maxcpus_per_cluster) + cpuid;
+ freq_core_reg = SCRATCH_FREQ_CORE_REG(data, mpidr_id);
+
+ *ndiv = readl(freq_core_reg) & NDIV_MASK;
+
+ return 0;
+}
+
+static void tegra234_set_cpu_ndiv(struct cpufreq_policy *policy, u64 ndiv)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ void __iomem *freq_core_reg;
+ u32 cpu, cpuid, clusterid;
+ u64 mpidr_id;
+
+ for_each_cpu_and(cpu, policy->cpus, cpu_online_mask) {
+ data->soc->ops->get_cpu_cluster_id(cpu, &cpuid, &clusterid);
+
+ /* use physical id to get address of per core frequency register */
+ mpidr_id = (clusterid * data->soc->maxcpus_per_cluster) + cpuid;
+ freq_core_reg = SCRATCH_FREQ_CORE_REG(data, mpidr_id);
+
+ writel(ndiv, freq_core_reg);
+ }
+}
+
+/*
+ * This register provides access to two counter values with a single
+ * 64-bit read. The counter values are used to determine the average
+ * actual frequency a core has run at over a period of time.
+ * [63:32] PLLP counter: Counts at fixed frequency (408 MHz)
+ * [31:0] Core clock counter: Counts on every core clock cycle
+ */
+static void tegra234_read_counters(struct tegra_cpu_ctr *c)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ void __iomem *actmon_reg;
+ u32 cpuid, clusterid;
+ u64 val;
+
+ data->soc->ops->get_cpu_cluster_id(c->cpu, &cpuid, &clusterid);
+ actmon_reg = CORE_ACTMON_CNTR_REG(data, clusterid, cpuid);
+
+ val = readq(actmon_reg);
+ c->last_refclk_cnt = upper_32_bits(val);
+ c->last_coreclk_cnt = lower_32_bits(val);
+ udelay(US_DELAY);
+ val = readq(actmon_reg);
+ c->refclk_cnt = upper_32_bits(val);
+ c->coreclk_cnt = lower_32_bits(val);
+}
+
+static struct tegra_cpufreq_ops tegra234_cpufreq_ops = {
+ .read_counters = tegra234_read_counters,
+ .get_cpu_cluster_id = tegra234_get_cpu_cluster_id,
+ .get_cpu_ndiv = tegra234_get_cpu_ndiv,
+ .set_cpu_ndiv = tegra234_set_cpu_ndiv,
+};
+
+static const struct tegra_cpufreq_soc tegra234_cpufreq_soc = {
+ .ops = &tegra234_cpufreq_ops,
+ .actmon_cntr_base = 0x9000,
+ .maxcpus_per_cluster = 4,
+ .num_clusters = 3,
+};
+
+static const struct tegra_cpufreq_soc tegra239_cpufreq_soc = {
+ .ops = &tegra234_cpufreq_ops,
+ .actmon_cntr_base = 0x4000,
+ .maxcpus_per_cluster = 8,
+ .num_clusters = 1,
+};
+
+static void tegra194_get_cpu_cluster_id(u32 cpu, u32 *cpuid, u32 *clusterid)
+{
+ u64 mpidr;
+
+ smp_call_function_single(cpu, tegra_get_cpu_mpidr, &mpidr, true);
+
+ if (cpuid)
+ *cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+ if (clusterid)
+ *clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+}
+
+/*
+ * Read per-core Read-only system register NVFREQ_FEEDBACK_EL1.
+ * The register provides frequency feedback information to
+ * determine the average actual frequency a core has run at over
+ * a period of time.
+ * [31:0] PLLP counter: Counts at fixed frequency (408 MHz)
+ * [63:32] Core clock counter: counts on every core clock cycle
+ * where the core is architecturally clocking
+ */
+static u64 read_freq_feedback(void)
+{
+ u64 val = 0;
+
+ asm volatile("mrs %0, s3_0_c15_c0_5" : "=r" (val) : );
+
+ return val;
+}
+
+static inline u32 map_ndiv_to_freq(struct mrq_cpu_ndiv_limits_response
+ *nltbl, u16 ndiv)
+{
+ return nltbl->ref_clk_hz / KHZ * ndiv / (nltbl->pdiv * nltbl->mdiv);
+}
+
+static void tegra194_read_counters(struct tegra_cpu_ctr *c)
+{
+ u64 val;
+
+ val = read_freq_feedback();
+ c->last_refclk_cnt = lower_32_bits(val);
+ c->last_coreclk_cnt = upper_32_bits(val);
+ udelay(US_DELAY);
+ val = read_freq_feedback();
+ c->refclk_cnt = lower_32_bits(val);
+ c->coreclk_cnt = upper_32_bits(val);
+}
+
+static void tegra_read_counters(struct work_struct *work)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ struct read_counters_work *read_counters_work;
+ struct tegra_cpu_ctr *c;
+
+ /*
+ * ref_clk_counter(32 bit counter) runs on constant clk,
+ * pll_p(408MHz).
+ * It will take = 2 ^ 32 / 408 MHz to overflow ref clk counter
+ * = 10526880 usec = 10.527 sec to overflow
+ *
+ * Like wise core_clk_counter(32 bit counter) runs on core clock.
+ * It's synchronized to crab_clk (cpu_crab_clk) which runs at
+ * freq of cluster. Assuming max cluster clock ~2000MHz,
+ * It will take = 2 ^ 32 / 2000 MHz to overflow core clk counter
+ * = ~2.147 sec to overflow
+ */
+ read_counters_work = container_of(work, struct read_counters_work,
+ work);
+ c = &read_counters_work->c;
+
+ data->soc->ops->read_counters(c);
+}
+
+/*
+ * Return instantaneous cpu speed
+ * Instantaneous freq is calculated as -
+ * -Takes sample on every query of getting the freq.
+ * - Read core and ref clock counters;
+ * - Delay for X us
+ * - Read above cycle counters again
+ * - Calculates freq by subtracting current and previous counters
+ * divided by the delay time or eqv. of ref_clk_counter in delta time
+ * - Return Kcycles/second, freq in KHz
+ *
+ * delta time period = x sec
+ * = delta ref_clk_counter / (408 * 10^6) sec
+ * freq in Hz = cycles/sec
+ * = (delta cycles / x sec
+ * = (delta cycles * 408 * 10^6) / delta ref_clk_counter
+ * in KHz = (delta cycles * 408 * 10^3) / delta ref_clk_counter
+ *
+ * @cpu - logical cpu whose freq to be updated
+ * Returns freq in KHz on success, 0 if cpu is offline
+ */
+static unsigned int tegra194_calculate_speed(u32 cpu)
+{
+ struct read_counters_work read_counters_work;
+ struct tegra_cpu_ctr c;
+ u32 delta_refcnt;
+ u32 delta_ccnt;
+ u32 rate_mhz;
+
+ /*
+ * udelay() is required to reconstruct cpu frequency over an
+ * observation window. Using workqueue to call udelay() with
+ * interrupts enabled.
+ */
+ read_counters_work.c.cpu = cpu;
+ INIT_WORK_ONSTACK(&read_counters_work.work, tegra_read_counters);
+ queue_work_on(cpu, read_counters_wq, &read_counters_work.work);
+ flush_work(&read_counters_work.work);
+ c = read_counters_work.c;
+
+ if (c.coreclk_cnt < c.last_coreclk_cnt)
+ delta_ccnt = c.coreclk_cnt + (MAX_CNT - c.last_coreclk_cnt);
+ else
+ delta_ccnt = c.coreclk_cnt - c.last_coreclk_cnt;
+ if (!delta_ccnt)
+ return 0;
+
+ /* ref clock is 32 bits */
+ if (c.refclk_cnt < c.last_refclk_cnt)
+ delta_refcnt = c.refclk_cnt + (MAX_CNT - c.last_refclk_cnt);
+ else
+ delta_refcnt = c.refclk_cnt - c.last_refclk_cnt;
+ if (!delta_refcnt) {
+ pr_debug("cpufreq: %d is idle, delta_refcnt: 0\n", cpu);
+ return 0;
+ }
+ rate_mhz = ((unsigned long)(delta_ccnt * REF_CLK_MHZ)) / delta_refcnt;
+
+ return (rate_mhz * KHZ); /* in KHz */
+}
+
+static void tegra194_get_cpu_ndiv_sysreg(void *ndiv)
+{
+ u64 ndiv_val;
+
+ asm volatile("mrs %0, s3_0_c15_c0_4" : "=r" (ndiv_val) : );
+
+ *(u64 *)ndiv = ndiv_val;
+}
+
+static int tegra194_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv)
+{
+ return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true);
+}
+
+static void tegra194_set_cpu_ndiv_sysreg(void *data)
+{
+ u64 ndiv_val = *(u64 *)data;
+
+ asm volatile("msr s3_0_c15_c0_4, %0" : : "r" (ndiv_val));
+}
+
+static void tegra194_set_cpu_ndiv(struct cpufreq_policy *policy, u64 ndiv)
+{
+ on_each_cpu_mask(policy->cpus, tegra194_set_cpu_ndiv_sysreg, &ndiv, true);
+}
+
+static unsigned int tegra194_get_speed(u32 cpu)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ struct cpufreq_frequency_table *pos;
+ u32 cpuid, clusterid;
+ unsigned int rate;
+ u64 ndiv;
+ int ret;
+
+ data->soc->ops->get_cpu_cluster_id(cpu, &cpuid, &clusterid);
+
+ /* reconstruct actual cpu freq using counters */
+ rate = tegra194_calculate_speed(cpu);
+
+ /* get last written ndiv value */
+ ret = data->soc->ops->get_cpu_ndiv(cpu, cpuid, clusterid, &ndiv);
+ if (WARN_ON_ONCE(ret))
+ return rate;
+
+ /*
+ * If the reconstructed frequency has acceptable delta from
+ * the last written value, then return freq corresponding
+ * to the last written ndiv value from freq_table. This is
+ * done to return consistent value.
+ */
+ cpufreq_for_each_valid_entry(pos, data->bpmp_luts[clusterid]) {
+ if (pos->driver_data != ndiv)
+ continue;
+
+ if (abs(pos->frequency - rate) > 115200) {
+ pr_warn("cpufreq: cpu%d,cur:%u,set:%u,set ndiv:%llu\n",
+ cpu, rate, pos->frequency, ndiv);
+ } else {
+ rate = pos->frequency;
+ }
+ break;
+ }
+ return rate;
+}
+
+static int tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy *policy,
+ struct cpufreq_frequency_table *bpmp_lut,
+ struct cpufreq_frequency_table **opp_table)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ struct cpufreq_frequency_table *freq_table = NULL;
+ struct cpufreq_frequency_table *pos;
+ struct device *cpu_dev;
+ struct dev_pm_opp *opp;
+ unsigned long rate;
+ int ret, max_opps;
+ int j = 0;
+
+ cpu_dev = get_cpu_device(policy->cpu);
+ if (!cpu_dev) {
+ pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu);
+ return -ENODEV;
+ }
+
+ /* Initialize OPP table mentioned in operating-points-v2 property in DT */
+ ret = dev_pm_opp_of_add_table_indexed(cpu_dev, 0);
+ if (!ret) {
+ max_opps = dev_pm_opp_get_opp_count(cpu_dev);
+ if (max_opps <= 0) {
+ dev_err(cpu_dev, "Failed to add OPPs\n");
+ return max_opps;
+ }
+
+ /* Disable all opps and cross-validate against LUT later */
+ for (rate = 0; ; rate++) {
+ opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate);
+ if (IS_ERR(opp))
+ break;
+
+ dev_pm_opp_put(opp);
+ dev_pm_opp_disable(cpu_dev, rate);
+ }
+ } else {
+ dev_err(cpu_dev, "Invalid or empty opp table in device tree\n");
+ data->icc_dram_bw_scaling = false;
+ return ret;
+ }
+
+ freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_KERNEL);
+ if (!freq_table)
+ return -ENOMEM;
+
+ /*
+ * Cross check the frequencies from BPMP-FW LUT against the OPP's present in DT.
+ * Enable only those DT OPP's which are present in LUT also.
+ */
+ cpufreq_for_each_valid_entry(pos, bpmp_lut) {
+ opp = dev_pm_opp_find_freq_exact(cpu_dev, pos->frequency * KHZ, false);
+ if (IS_ERR(opp))
+ continue;
+
+ dev_pm_opp_put(opp);
+
+ ret = dev_pm_opp_enable(cpu_dev, pos->frequency * KHZ);
+ if (ret < 0)
+ return ret;
+
+ freq_table[j].driver_data = pos->driver_data;
+ freq_table[j].frequency = pos->frequency;
+ j++;
+ }
+
+ freq_table[j].driver_data = pos->driver_data;
+ freq_table[j].frequency = CPUFREQ_TABLE_END;
+
+ *opp_table = &freq_table[0];
+
+ dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
+
+ return ret;
+}
+
+static int tegra194_cpufreq_init(struct cpufreq_policy *policy)
+{
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+ int maxcpus_per_cluster = data->soc->maxcpus_per_cluster;
+ struct cpufreq_frequency_table *freq_table;
+ struct cpufreq_frequency_table *bpmp_lut;
+ u32 start_cpu, cpu;
+ u32 clusterid;
+ int ret;
+
+ data->soc->ops->get_cpu_cluster_id(policy->cpu, NULL, &clusterid);
+ if (clusterid >= data->soc->num_clusters || !data->bpmp_luts[clusterid])
+ return -EINVAL;
+
+ start_cpu = rounddown(policy->cpu, maxcpus_per_cluster);
+ /* set same policy for all cpus in a cluster */
+ for (cpu = start_cpu; cpu < (start_cpu + maxcpus_per_cluster); cpu++) {
+ if (cpu_possible(cpu))
+ cpumask_set_cpu(cpu, policy->cpus);
+ }
+ policy->cpuinfo.transition_latency = TEGRA_CPUFREQ_TRANSITION_LATENCY;
+
+ bpmp_lut = data->bpmp_luts[clusterid];
+
+ if (data->icc_dram_bw_scaling) {
+ ret = tegra_cpufreq_init_cpufreq_table(policy, bpmp_lut, &freq_table);
+ if (!ret) {
+ policy->freq_table = freq_table;
+ return 0;
+ }
+ }
+
+ data->icc_dram_bw_scaling = false;
+ policy->freq_table = bpmp_lut;
+ pr_info("OPP tables missing from DT, EMC frequency scaling disabled\n");
+
+ return 0;
+}
+
+static int tegra194_cpufreq_online(struct cpufreq_policy *policy)
+{
+ /* We did light-weight tear down earlier, nothing to do here */
+ return 0;
+}
+
+static int tegra194_cpufreq_offline(struct cpufreq_policy *policy)
+{
+ /*
+ * Preserve policy->driver_data and don't free resources on light-weight
+ * tear down.
+ */
+
+ return 0;
+}
+
+static int tegra194_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ struct device *cpu_dev = get_cpu_device(policy->cpu);
+
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+ dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);
+
+ return 0;
+}
+
+static int tegra194_cpufreq_set_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ struct cpufreq_frequency_table *tbl = policy->freq_table + index;
+ struct tegra194_cpufreq_data *data = cpufreq_get_driver_data();
+
+ /*
+ * Each core writes frequency in per core register. Then both cores
+ * in a cluster run at same frequency which is the maximum frequency
+ * request out of the values requested by both cores in that cluster.
+ */
+ data->soc->ops->set_cpu_ndiv(policy, (u64)tbl->driver_data);
+
+ if (data->icc_dram_bw_scaling)
+ tegra_cpufreq_set_bw(policy, tbl->frequency);
+
+ return 0;
+}
+
+static struct cpufreq_driver tegra194_cpufreq_driver = {
+ .name = "tegra194",
+ .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+ CPUFREQ_IS_COOLING_DEV,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = tegra194_cpufreq_set_target,
+ .get = tegra194_get_speed,
+ .init = tegra194_cpufreq_init,
+ .exit = tegra194_cpufreq_exit,
+ .online = tegra194_cpufreq_online,
+ .offline = tegra194_cpufreq_offline,
+ .attr = cpufreq_generic_attr,
+};
+
+static struct tegra_cpufreq_ops tegra194_cpufreq_ops = {
+ .read_counters = tegra194_read_counters,
+ .get_cpu_cluster_id = tegra194_get_cpu_cluster_id,
+ .get_cpu_ndiv = tegra194_get_cpu_ndiv,
+ .set_cpu_ndiv = tegra194_set_cpu_ndiv,
+};
+
+static const struct tegra_cpufreq_soc tegra194_cpufreq_soc = {
+ .ops = &tegra194_cpufreq_ops,
+ .maxcpus_per_cluster = 2,
+ .num_clusters = 4,
+};
+
+static void tegra194_cpufreq_free_resources(void)
+{
+ destroy_workqueue(read_counters_wq);
+}
+
+static struct cpufreq_frequency_table *
+tegra_cpufreq_bpmp_read_lut(struct platform_device *pdev, struct tegra_bpmp *bpmp,
+ unsigned int cluster_id)
+{
+ struct cpufreq_frequency_table *freq_table;
+ struct mrq_cpu_ndiv_limits_response resp;
+ unsigned int num_freqs, ndiv, delta_ndiv;
+ struct mrq_cpu_ndiv_limits_request req;
+ struct tegra_bpmp_message msg;
+ u16 freq_table_step_size;
+ int err, index;
+
+ memset(&req, 0, sizeof(req));
+ req.cluster_id = cluster_id;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.mrq = MRQ_CPU_NDIV_LIMITS;
+ msg.tx.data = &req;
+ msg.tx.size = sizeof(req);
+ msg.rx.data = &resp;
+ msg.rx.size = sizeof(resp);
+
+ err = tegra_bpmp_transfer(bpmp, &msg);
+ if (err)
+ return ERR_PTR(err);
+ if (msg.rx.ret == -BPMP_EINVAL) {
+ /* Cluster not available */
+ return NULL;
+ }
+ if (msg.rx.ret)
+ return ERR_PTR(-EINVAL);
+
+ /*
+ * Make sure frequency table step is a multiple of mdiv to match
+ * vhint table granularity.
+ */
+ freq_table_step_size = resp.mdiv *
+ DIV_ROUND_UP(CPUFREQ_TBL_STEP_HZ, resp.ref_clk_hz);
+
+ dev_dbg(&pdev->dev, "cluster %d: frequency table step size: %d\n",
+ cluster_id, freq_table_step_size);
+
+ delta_ndiv = resp.ndiv_max - resp.ndiv_min;
+
+ if (unlikely(delta_ndiv == 0)) {
+ num_freqs = 1;
+ } else {
+ /* We store both ndiv_min and ndiv_max hence the +1 */
+ num_freqs = delta_ndiv / freq_table_step_size + 1;
+ }
+
+ num_freqs += (delta_ndiv % freq_table_step_size) ? 1 : 0;
+
+ freq_table = devm_kcalloc(&pdev->dev, num_freqs + 1,
+ sizeof(*freq_table), GFP_KERNEL);
+ if (!freq_table)
+ return ERR_PTR(-ENOMEM);
+
+ for (index = 0, ndiv = resp.ndiv_min;
+ ndiv < resp.ndiv_max;
+ index++, ndiv += freq_table_step_size) {
+ freq_table[index].driver_data = ndiv;
+ freq_table[index].frequency = map_ndiv_to_freq(&resp, ndiv);
+ }
+
+ freq_table[index].driver_data = resp.ndiv_max;
+ freq_table[index++].frequency = map_ndiv_to_freq(&resp, resp.ndiv_max);
+ freq_table[index].frequency = CPUFREQ_TABLE_END;
+
+ return freq_table;
+}
+
+static int tegra194_cpufreq_probe(struct platform_device *pdev)
+{
+ const struct tegra_cpufreq_soc *soc;
+ struct tegra194_cpufreq_data *data;
+ struct tegra_bpmp *bpmp;
+ struct device *cpu_dev;
+ int err, i;
+
+ data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ soc = of_device_get_match_data(&pdev->dev);
+
+ if (soc->ops && soc->maxcpus_per_cluster && soc->num_clusters) {
+ data->soc = soc;
+ } else {
+ dev_err(&pdev->dev, "soc data missing\n");
+ return -EINVAL;
+ }
+
+ data->bpmp_luts = devm_kcalloc(&pdev->dev, data->soc->num_clusters,
+ sizeof(*data->bpmp_luts), GFP_KERNEL);
+ if (!data->bpmp_luts)
+ return -ENOMEM;
+
+ if (soc->actmon_cntr_base) {
+ /* mmio registers are used for frequency request and re-construction */
+ data->regs = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(data->regs))
+ return PTR_ERR(data->regs);
+ }
+
+ platform_set_drvdata(pdev, data);
+
+ bpmp = tegra_bpmp_get(&pdev->dev);
+ if (IS_ERR(bpmp))
+ return PTR_ERR(bpmp);
+
+ read_counters_wq = alloc_workqueue("read_counters_wq", __WQ_LEGACY, 1);
+ if (!read_counters_wq) {
+ dev_err(&pdev->dev, "fail to create_workqueue\n");
+ err = -EINVAL;
+ goto put_bpmp;
+ }
+
+ for (i = 0; i < data->soc->num_clusters; i++) {
+ data->bpmp_luts[i] = tegra_cpufreq_bpmp_read_lut(pdev, bpmp, i);
+ if (IS_ERR(data->bpmp_luts[i])) {
+ err = PTR_ERR(data->bpmp_luts[i]);
+ goto err_free_res;
+ }
+ }
+
+ tegra194_cpufreq_driver.driver_data = data;
+
+ /* Check for optional OPPv2 and interconnect paths on CPU0 to enable ICC scaling */
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev) {
+ err = -EPROBE_DEFER;
+ goto err_free_res;
+ }
+
+ if (dev_pm_opp_of_get_opp_desc_node(cpu_dev)) {
+ err = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL);
+ if (!err)
+ data->icc_dram_bw_scaling = true;
+ }
+
+ err = cpufreq_register_driver(&tegra194_cpufreq_driver);
+ if (!err)
+ goto put_bpmp;
+
+err_free_res:
+ tegra194_cpufreq_free_resources();
+put_bpmp:
+ tegra_bpmp_put(bpmp);
+ return err;
+}
+
+static void tegra194_cpufreq_remove(struct platform_device *pdev)
+{
+ cpufreq_unregister_driver(&tegra194_cpufreq_driver);
+ tegra194_cpufreq_free_resources();
+}
+
+static const struct of_device_id tegra194_cpufreq_of_match[] = {
+ { .compatible = "nvidia,tegra194-ccplex", .data = &tegra194_cpufreq_soc },
+ { .compatible = "nvidia,tegra234-ccplex-cluster", .data = &tegra234_cpufreq_soc },
+ { .compatible = "nvidia,tegra239-ccplex-cluster", .data = &tegra239_cpufreq_soc },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, tegra194_cpufreq_of_match);
+
+static struct platform_driver tegra194_ccplex_driver = {
+ .driver = {
+ .name = "tegra194-cpufreq",
+ .of_match_table = tegra194_cpufreq_of_match,
+ },
+ .probe = tegra194_cpufreq_probe,
+ .remove_new = tegra194_cpufreq_remove,
+};
+module_platform_driver(tegra194_ccplex_driver);
+
+MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>");
+MODULE_AUTHOR("Sumit Gupta <sumitg@nvidia.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra194 cpufreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c
new file mode 100644
index 0000000000..a573186704
--- /dev/null
+++ b/drivers/cpufreq/tegra20-cpufreq.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2010 Google, Inc.
+ *
+ * Author:
+ * Colin Cross <ccross@google.com>
+ * Based on arch/arm/plat-omap/cpu-omap.c, (C) 2005 Nokia Corporation
+ */
+
+#include <linux/bits.h>
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/types.h>
+
+#include <soc/tegra/common.h>
+#include <soc/tegra/fuse.h>
+
+static bool cpu0_node_has_opp_v2_prop(void)
+{
+ struct device_node *np = of_cpu_device_node_get(0);
+ bool ret = false;
+
+ if (of_property_present(np, "operating-points-v2"))
+ ret = true;
+
+ of_node_put(np);
+ return ret;
+}
+
+static void tegra20_cpufreq_put_supported_hw(void *opp_token)
+{
+ dev_pm_opp_put_supported_hw((unsigned long) opp_token);
+}
+
+static void tegra20_cpufreq_dt_unregister(void *cpufreq_dt)
+{
+ platform_device_unregister(cpufreq_dt);
+}
+
+static int tegra20_cpufreq_probe(struct platform_device *pdev)
+{
+ struct platform_device *cpufreq_dt;
+ struct device *cpu_dev;
+ u32 versions[2];
+ int err;
+
+ if (!cpu0_node_has_opp_v2_prop()) {
+ dev_err(&pdev->dev, "operating points not found\n");
+ dev_err(&pdev->dev, "please update your device tree\n");
+ return -ENODEV;
+ }
+
+ if (of_machine_is_compatible("nvidia,tegra20")) {
+ versions[0] = BIT(tegra_sku_info.cpu_process_id);
+ versions[1] = BIT(tegra_sku_info.soc_speedo_id);
+ } else {
+ versions[0] = BIT(tegra_sku_info.cpu_process_id);
+ versions[1] = BIT(tegra_sku_info.cpu_speedo_id);
+ }
+
+ dev_info(&pdev->dev, "hardware version 0x%x 0x%x\n",
+ versions[0], versions[1]);
+
+ cpu_dev = get_cpu_device(0);
+ if (WARN_ON(!cpu_dev))
+ return -ENODEV;
+
+ err = dev_pm_opp_set_supported_hw(cpu_dev, versions, 2);
+ if (err < 0) {
+ dev_err(&pdev->dev, "failed to set supported hw: %d\n", err);
+ return err;
+ }
+
+ err = devm_add_action_or_reset(&pdev->dev,
+ tegra20_cpufreq_put_supported_hw,
+ (void *)((unsigned long) err));
+ if (err)
+ return err;
+
+ cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+ err = PTR_ERR_OR_ZERO(cpufreq_dt);
+ if (err) {
+ dev_err(&pdev->dev,
+ "failed to create cpufreq-dt device: %d\n", err);
+ return err;
+ }
+
+ err = devm_add_action_or_reset(&pdev->dev,
+ tegra20_cpufreq_dt_unregister,
+ cpufreq_dt);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+static struct platform_driver tegra20_cpufreq_driver = {
+ .probe = tegra20_cpufreq_probe,
+ .driver = {
+ .name = "tegra20-cpufreq",
+ },
+};
+module_platform_driver(tegra20_cpufreq_driver);
+
+MODULE_ALIAS("platform:tegra20-cpufreq");
+MODULE_AUTHOR("Colin Cross <ccross@android.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra20 cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c
new file mode 100644
index 0000000000..3c37d78996
--- /dev/null
+++ b/drivers/cpufreq/ti-cpufreq.c
@@ -0,0 +1,462 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * TI CPUFreq/OPP hw-supported driver
+ *
+ * Copyright (C) 2016-2017 Texas Instruments, Inc.
+ * Dave Gerlach <d-gerlach@ti.com>
+ */
+
+#include <linux/cpu.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define REVISION_MASK 0xF
+#define REVISION_SHIFT 28
+
+#define AM33XX_800M_ARM_MPU_MAX_FREQ 0x1E2F
+#define AM43XX_600M_ARM_MPU_MAX_FREQ 0xFFA
+
+#define DRA7_EFUSE_HAS_OD_MPU_OPP 11
+#define DRA7_EFUSE_HAS_HIGH_MPU_OPP 15
+#define DRA76_EFUSE_HAS_PLUS_MPU_OPP 18
+#define DRA7_EFUSE_HAS_ALL_MPU_OPP 23
+#define DRA76_EFUSE_HAS_ALL_MPU_OPP 24
+
+#define DRA7_EFUSE_NOM_MPU_OPP BIT(0)
+#define DRA7_EFUSE_OD_MPU_OPP BIT(1)
+#define DRA7_EFUSE_HIGH_MPU_OPP BIT(2)
+#define DRA76_EFUSE_PLUS_MPU_OPP BIT(3)
+
+#define OMAP3_CONTROL_DEVICE_STATUS 0x4800244C
+#define OMAP3_CONTROL_IDCODE 0x4830A204
+#define OMAP34xx_ProdID_SKUID 0x4830A20C
+#define OMAP3_SYSCON_BASE (0x48000000 + 0x2000 + 0x270)
+
+#define AM625_EFUSE_K_MPU_OPP 11
+#define AM625_EFUSE_S_MPU_OPP 19
+#define AM625_EFUSE_T_MPU_OPP 20
+
+#define AM625_SUPPORT_K_MPU_OPP BIT(0)
+#define AM625_SUPPORT_S_MPU_OPP BIT(1)
+#define AM625_SUPPORT_T_MPU_OPP BIT(2)
+
+#define VERSION_COUNT 2
+
+struct ti_cpufreq_data;
+
+struct ti_cpufreq_soc_data {
+ const char * const *reg_names;
+ unsigned long (*efuse_xlate)(struct ti_cpufreq_data *opp_data,
+ unsigned long efuse);
+ unsigned long efuse_fallback;
+ unsigned long efuse_offset;
+ unsigned long efuse_mask;
+ unsigned long efuse_shift;
+ unsigned long rev_offset;
+ bool multi_regulator;
+};
+
+struct ti_cpufreq_data {
+ struct device *cpu_dev;
+ struct device_node *opp_node;
+ struct regmap *syscon;
+ const struct ti_cpufreq_soc_data *soc_data;
+};
+
+static unsigned long amx3_efuse_xlate(struct ti_cpufreq_data *opp_data,
+ unsigned long efuse)
+{
+ if (!efuse)
+ efuse = opp_data->soc_data->efuse_fallback;
+ /* AM335x and AM437x use "OPP disable" bits, so invert */
+ return ~efuse;
+}
+
+static unsigned long dra7_efuse_xlate(struct ti_cpufreq_data *opp_data,
+ unsigned long efuse)
+{
+ unsigned long calculated_efuse = DRA7_EFUSE_NOM_MPU_OPP;
+
+ /*
+ * The efuse on dra7 and am57 parts contains a specific
+ * value indicating the highest available OPP.
+ */
+
+ switch (efuse) {
+ case DRA76_EFUSE_HAS_PLUS_MPU_OPP:
+ case DRA76_EFUSE_HAS_ALL_MPU_OPP:
+ calculated_efuse |= DRA76_EFUSE_PLUS_MPU_OPP;
+ fallthrough;
+ case DRA7_EFUSE_HAS_ALL_MPU_OPP:
+ case DRA7_EFUSE_HAS_HIGH_MPU_OPP:
+ calculated_efuse |= DRA7_EFUSE_HIGH_MPU_OPP;
+ fallthrough;
+ case DRA7_EFUSE_HAS_OD_MPU_OPP:
+ calculated_efuse |= DRA7_EFUSE_OD_MPU_OPP;
+ }
+
+ return calculated_efuse;
+}
+
+static unsigned long omap3_efuse_xlate(struct ti_cpufreq_data *opp_data,
+ unsigned long efuse)
+{
+ /* OPP enable bit ("Speed Binned") */
+ return BIT(efuse);
+}
+
+static unsigned long am625_efuse_xlate(struct ti_cpufreq_data *opp_data,
+ unsigned long efuse)
+{
+ unsigned long calculated_efuse = AM625_SUPPORT_K_MPU_OPP;
+
+ switch (efuse) {
+ case AM625_EFUSE_T_MPU_OPP:
+ calculated_efuse |= AM625_SUPPORT_T_MPU_OPP;
+ fallthrough;
+ case AM625_EFUSE_S_MPU_OPP:
+ calculated_efuse |= AM625_SUPPORT_S_MPU_OPP;
+ fallthrough;
+ case AM625_EFUSE_K_MPU_OPP:
+ calculated_efuse |= AM625_SUPPORT_K_MPU_OPP;
+ }
+
+ return calculated_efuse;
+}
+
+static struct ti_cpufreq_soc_data am3x_soc_data = {
+ .efuse_xlate = amx3_efuse_xlate,
+ .efuse_fallback = AM33XX_800M_ARM_MPU_MAX_FREQ,
+ .efuse_offset = 0x07fc,
+ .efuse_mask = 0x1fff,
+ .rev_offset = 0x600,
+ .multi_regulator = false,
+};
+
+static struct ti_cpufreq_soc_data am4x_soc_data = {
+ .efuse_xlate = amx3_efuse_xlate,
+ .efuse_fallback = AM43XX_600M_ARM_MPU_MAX_FREQ,
+ .efuse_offset = 0x0610,
+ .efuse_mask = 0x3f,
+ .rev_offset = 0x600,
+ .multi_regulator = false,
+};
+
+static struct ti_cpufreq_soc_data dra7_soc_data = {
+ .efuse_xlate = dra7_efuse_xlate,
+ .efuse_offset = 0x020c,
+ .efuse_mask = 0xf80000,
+ .efuse_shift = 19,
+ .rev_offset = 0x204,
+ .multi_regulator = true,
+};
+
+/*
+ * OMAP35x TRM (SPRUF98K):
+ * CONTROL_IDCODE (0x4830 A204) describes Silicon revisions.
+ * Control OMAP Status Register 15:0 (Address 0x4800 244C)
+ * to separate between omap3503, omap3515, omap3525, omap3530
+ * and feature presence.
+ * There are encodings for versions limited to 400/266MHz
+ * but we ignore.
+ * Not clear if this also holds for omap34xx.
+ * some eFuse values e.g. CONTROL_FUSE_OPP1_VDD1
+ * are stored in the SYSCON register range
+ * Register 0x4830A20C [ProdID.SKUID] [0:3]
+ * 0x0 for normal 600/430MHz device.
+ * 0x8 for 720/520MHz device.
+ * Not clear what omap34xx value is.
+ */
+
+static struct ti_cpufreq_soc_data omap34xx_soc_data = {
+ .efuse_xlate = omap3_efuse_xlate,
+ .efuse_offset = OMAP34xx_ProdID_SKUID - OMAP3_SYSCON_BASE,
+ .efuse_shift = 3,
+ .efuse_mask = BIT(3),
+ .rev_offset = OMAP3_CONTROL_IDCODE - OMAP3_SYSCON_BASE,
+ .multi_regulator = false,
+};
+
+/*
+ * AM/DM37x TRM (SPRUGN4M)
+ * CONTROL_IDCODE (0x4830 A204) describes Silicon revisions.
+ * Control Device Status Register 15:0 (Address 0x4800 244C)
+ * to separate between am3703, am3715, dm3725, dm3730
+ * and feature presence.
+ * Speed Binned = Bit 9
+ * 0 800/600 MHz
+ * 1 1000/800 MHz
+ * some eFuse values e.g. CONTROL_FUSE_OPP 1G_VDD1
+ * are stored in the SYSCON register range.
+ * There is no 0x4830A20C [ProdID.SKUID] register (exists but
+ * seems to always read as 0).
+ */
+
+static const char * const omap3_reg_names[] = {"cpu0", "vbb", NULL};
+
+static struct ti_cpufreq_soc_data omap36xx_soc_data = {
+ .reg_names = omap3_reg_names,
+ .efuse_xlate = omap3_efuse_xlate,
+ .efuse_offset = OMAP3_CONTROL_DEVICE_STATUS - OMAP3_SYSCON_BASE,
+ .efuse_shift = 9,
+ .efuse_mask = BIT(9),
+ .rev_offset = OMAP3_CONTROL_IDCODE - OMAP3_SYSCON_BASE,
+ .multi_regulator = true,
+};
+
+/*
+ * AM3517 is quite similar to AM/DM37x except that it has no
+ * high speed grade eFuse and no abb ldo
+ */
+
+static struct ti_cpufreq_soc_data am3517_soc_data = {
+ .efuse_xlate = omap3_efuse_xlate,
+ .efuse_offset = OMAP3_CONTROL_DEVICE_STATUS - OMAP3_SYSCON_BASE,
+ .efuse_shift = 0,
+ .efuse_mask = 0,
+ .rev_offset = OMAP3_CONTROL_IDCODE - OMAP3_SYSCON_BASE,
+ .multi_regulator = false,
+};
+
+static struct ti_cpufreq_soc_data am625_soc_data = {
+ .efuse_xlate = am625_efuse_xlate,
+ .efuse_offset = 0x0018,
+ .efuse_mask = 0x07c0,
+ .efuse_shift = 0x6,
+ .rev_offset = 0x0014,
+ .multi_regulator = false,
+};
+
+/**
+ * ti_cpufreq_get_efuse() - Parse and return efuse value present on SoC
+ * @opp_data: pointer to ti_cpufreq_data context
+ * @efuse_value: Set to the value parsed from efuse
+ *
+ * Returns error code if efuse not read properly.
+ */
+static int ti_cpufreq_get_efuse(struct ti_cpufreq_data *opp_data,
+ u32 *efuse_value)
+{
+ struct device *dev = opp_data->cpu_dev;
+ u32 efuse;
+ int ret;
+
+ ret = regmap_read(opp_data->syscon, opp_data->soc_data->efuse_offset,
+ &efuse);
+ if (ret == -EIO) {
+ /* not a syscon register! */
+ void __iomem *regs = ioremap(OMAP3_SYSCON_BASE +
+ opp_data->soc_data->efuse_offset, 4);
+
+ if (!regs)
+ return -ENOMEM;
+ efuse = readl(regs);
+ iounmap(regs);
+ }
+ else if (ret) {
+ dev_err(dev,
+ "Failed to read the efuse value from syscon: %d\n",
+ ret);
+ return ret;
+ }
+
+ efuse = (efuse & opp_data->soc_data->efuse_mask);
+ efuse >>= opp_data->soc_data->efuse_shift;
+
+ *efuse_value = opp_data->soc_data->efuse_xlate(opp_data, efuse);
+
+ return 0;
+}
+
+/**
+ * ti_cpufreq_get_rev() - Parse and return rev value present on SoC
+ * @opp_data: pointer to ti_cpufreq_data context
+ * @revision_value: Set to the value parsed from revision register
+ *
+ * Returns error code if revision not read properly.
+ */
+static int ti_cpufreq_get_rev(struct ti_cpufreq_data *opp_data,
+ u32 *revision_value)
+{
+ struct device *dev = opp_data->cpu_dev;
+ u32 revision;
+ int ret;
+
+ ret = regmap_read(opp_data->syscon, opp_data->soc_data->rev_offset,
+ &revision);
+ if (ret == -EIO) {
+ /* not a syscon register! */
+ void __iomem *regs = ioremap(OMAP3_SYSCON_BASE +
+ opp_data->soc_data->rev_offset, 4);
+
+ if (!regs)
+ return -ENOMEM;
+ revision = readl(regs);
+ iounmap(regs);
+ }
+ else if (ret) {
+ dev_err(dev,
+ "Failed to read the revision number from syscon: %d\n",
+ ret);
+ return ret;
+ }
+
+ *revision_value = BIT((revision >> REVISION_SHIFT) & REVISION_MASK);
+
+ return 0;
+}
+
+static int ti_cpufreq_setup_syscon_register(struct ti_cpufreq_data *opp_data)
+{
+ struct device *dev = opp_data->cpu_dev;
+ struct device_node *np = opp_data->opp_node;
+
+ opp_data->syscon = syscon_regmap_lookup_by_phandle(np,
+ "syscon");
+ if (IS_ERR(opp_data->syscon)) {
+ dev_err(dev,
+ "\"syscon\" is missing, cannot use OPPv2 table.\n");
+ return PTR_ERR(opp_data->syscon);
+ }
+
+ return 0;
+}
+
+static const struct of_device_id ti_cpufreq_of_match[] = {
+ { .compatible = "ti,am33xx", .data = &am3x_soc_data, },
+ { .compatible = "ti,am3517", .data = &am3517_soc_data, },
+ { .compatible = "ti,am43", .data = &am4x_soc_data, },
+ { .compatible = "ti,dra7", .data = &dra7_soc_data },
+ { .compatible = "ti,omap34xx", .data = &omap34xx_soc_data, },
+ { .compatible = "ti,omap36xx", .data = &omap36xx_soc_data, },
+ { .compatible = "ti,am625", .data = &am625_soc_data, },
+ { .compatible = "ti,am62a7", .data = &am625_soc_data, },
+ /* legacy */
+ { .compatible = "ti,omap3430", .data = &omap34xx_soc_data, },
+ { .compatible = "ti,omap3630", .data = &omap36xx_soc_data, },
+ {},
+};
+
+static const struct of_device_id *ti_cpufreq_match_node(void)
+{
+ struct device_node *np;
+ const struct of_device_id *match;
+
+ np = of_find_node_by_path("/");
+ match = of_match_node(ti_cpufreq_of_match, np);
+ of_node_put(np);
+
+ return match;
+}
+
+static int ti_cpufreq_probe(struct platform_device *pdev)
+{
+ u32 version[VERSION_COUNT];
+ const struct of_device_id *match;
+ struct ti_cpufreq_data *opp_data;
+ const char * const default_reg_names[] = {"vdd", "vbb", NULL};
+ int ret;
+ struct dev_pm_opp_config config = {
+ .supported_hw = version,
+ .supported_hw_count = ARRAY_SIZE(version),
+ };
+
+ match = dev_get_platdata(&pdev->dev);
+ if (!match)
+ return -ENODEV;
+
+ opp_data = devm_kzalloc(&pdev->dev, sizeof(*opp_data), GFP_KERNEL);
+ if (!opp_data)
+ return -ENOMEM;
+
+ opp_data->soc_data = match->data;
+
+ opp_data->cpu_dev = get_cpu_device(0);
+ if (!opp_data->cpu_dev) {
+ pr_err("%s: Failed to get device for CPU0\n", __func__);
+ return -ENODEV;
+ }
+
+ opp_data->opp_node = dev_pm_opp_of_get_opp_desc_node(opp_data->cpu_dev);
+ if (!opp_data->opp_node) {
+ dev_info(opp_data->cpu_dev,
+ "OPP-v2 not supported, cpufreq-dt will attempt to use legacy tables.\n");
+ goto register_cpufreq_dt;
+ }
+
+ ret = ti_cpufreq_setup_syscon_register(opp_data);
+ if (ret)
+ goto fail_put_node;
+
+ /*
+ * OPPs determine whether or not they are supported based on
+ * two metrics:
+ * 0 - SoC Revision
+ * 1 - eFuse value
+ */
+ ret = ti_cpufreq_get_rev(opp_data, &version[0]);
+ if (ret)
+ goto fail_put_node;
+
+ ret = ti_cpufreq_get_efuse(opp_data, &version[1]);
+ if (ret)
+ goto fail_put_node;
+
+ if (opp_data->soc_data->multi_regulator) {
+ if (opp_data->soc_data->reg_names)
+ config.regulator_names = opp_data->soc_data->reg_names;
+ else
+ config.regulator_names = default_reg_names;
+ }
+
+ ret = dev_pm_opp_set_config(opp_data->cpu_dev, &config);
+ if (ret < 0) {
+ dev_err(opp_data->cpu_dev, "Failed to set OPP config\n");
+ goto fail_put_node;
+ }
+
+ of_node_put(opp_data->opp_node);
+
+register_cpufreq_dt:
+ platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+
+ return 0;
+
+fail_put_node:
+ of_node_put(opp_data->opp_node);
+
+ return ret;
+}
+
+static int __init ti_cpufreq_init(void)
+{
+ const struct of_device_id *match;
+
+ /* Check to ensure we are on a compatible platform */
+ match = ti_cpufreq_match_node();
+ if (match)
+ platform_device_register_data(NULL, "ti-cpufreq", -1, match,
+ sizeof(*match));
+
+ return 0;
+}
+module_init(ti_cpufreq_init);
+
+static struct platform_driver ti_cpufreq_driver = {
+ .probe = ti_cpufreq_probe,
+ .driver = {
+ .name = "ti-cpufreq",
+ },
+};
+builtin_platform_driver(ti_cpufreq_driver);
+
+MODULE_DESCRIPTION("TI CPUFreq/OPP hw-supported driver");
+MODULE_AUTHOR("Dave Gerlach <d-gerlach@ti.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c
new file mode 100644
index 0000000000..9ac4ea50b8
--- /dev/null
+++ b/drivers/cpufreq/vexpress-spc-cpufreq.c
@@ -0,0 +1,577 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Versatile Express SPC CPUFreq Interface driver
+ *
+ * Copyright (C) 2013 - 2019 ARM Ltd.
+ * Sudeep Holla <sudeep.holla@arm.com>
+ *
+ * Copyright (C) 2013 Linaro.
+ * Viresh Kumar <viresh.kumar@linaro.org>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+#include <linux/topology.h>
+#include <linux/types.h>
+
+/* Currently we support only two clusters */
+#define A15_CLUSTER 0
+#define A7_CLUSTER 1
+#define MAX_CLUSTERS 2
+
+#ifdef CONFIG_BL_SWITCHER
+#include <asm/bL_switcher.h>
+static bool bL_switching_enabled;
+#define is_bL_switching_enabled() bL_switching_enabled
+#define set_switching_enabled(x) (bL_switching_enabled = (x))
+#else
+#define is_bL_switching_enabled() false
+#define set_switching_enabled(x) do { } while (0)
+#define bL_switch_request(...) do { } while (0)
+#define bL_switcher_put_enabled() do { } while (0)
+#define bL_switcher_get_enabled() do { } while (0)
+#endif
+
+#define ACTUAL_FREQ(cluster, freq) ((cluster == A7_CLUSTER) ? freq << 1 : freq)
+#define VIRT_FREQ(cluster, freq) ((cluster == A7_CLUSTER) ? freq >> 1 : freq)
+
+static struct clk *clk[MAX_CLUSTERS];
+static struct cpufreq_frequency_table *freq_table[MAX_CLUSTERS + 1];
+static atomic_t cluster_usage[MAX_CLUSTERS + 1];
+
+static unsigned int clk_big_min; /* (Big) clock frequencies */
+static unsigned int clk_little_max; /* Maximum clock frequency (Little) */
+
+static DEFINE_PER_CPU(unsigned int, physical_cluster);
+static DEFINE_PER_CPU(unsigned int, cpu_last_req_freq);
+
+static struct mutex cluster_lock[MAX_CLUSTERS];
+
+static inline int raw_cpu_to_cluster(int cpu)
+{
+ return topology_physical_package_id(cpu);
+}
+
+static inline int cpu_to_cluster(int cpu)
+{
+ return is_bL_switching_enabled() ?
+ MAX_CLUSTERS : raw_cpu_to_cluster(cpu);
+}
+
+static unsigned int find_cluster_maxfreq(int cluster)
+{
+ int j;
+ u32 max_freq = 0, cpu_freq;
+
+ for_each_online_cpu(j) {
+ cpu_freq = per_cpu(cpu_last_req_freq, j);
+
+ if (cluster == per_cpu(physical_cluster, j) &&
+ max_freq < cpu_freq)
+ max_freq = cpu_freq;
+ }
+
+ return max_freq;
+}
+
+static unsigned int clk_get_cpu_rate(unsigned int cpu)
+{
+ u32 cur_cluster = per_cpu(physical_cluster, cpu);
+ u32 rate = clk_get_rate(clk[cur_cluster]) / 1000;
+
+ /* For switcher we use virtual A7 clock rates */
+ if (is_bL_switching_enabled())
+ rate = VIRT_FREQ(cur_cluster, rate);
+
+ return rate;
+}
+
+static unsigned int ve_spc_cpufreq_get_rate(unsigned int cpu)
+{
+ if (is_bL_switching_enabled())
+ return per_cpu(cpu_last_req_freq, cpu);
+ else
+ return clk_get_cpu_rate(cpu);
+}
+
+static unsigned int
+ve_spc_cpufreq_set_rate(u32 cpu, u32 old_cluster, u32 new_cluster, u32 rate)
+{
+ u32 new_rate, prev_rate;
+ int ret;
+ bool bLs = is_bL_switching_enabled();
+
+ mutex_lock(&cluster_lock[new_cluster]);
+
+ if (bLs) {
+ prev_rate = per_cpu(cpu_last_req_freq, cpu);
+ per_cpu(cpu_last_req_freq, cpu) = rate;
+ per_cpu(physical_cluster, cpu) = new_cluster;
+
+ new_rate = find_cluster_maxfreq(new_cluster);
+ new_rate = ACTUAL_FREQ(new_cluster, new_rate);
+ } else {
+ new_rate = rate;
+ }
+
+ ret = clk_set_rate(clk[new_cluster], new_rate * 1000);
+ if (!ret) {
+ /*
+ * FIXME: clk_set_rate hasn't returned an error here however it
+ * may be that clk_change_rate failed due to hardware or
+ * firmware issues and wasn't able to report that due to the
+ * current design of the clk core layer. To work around this
+ * problem we will read back the clock rate and check it is
+ * correct. This needs to be removed once clk core is fixed.
+ */
+ if (clk_get_rate(clk[new_cluster]) != new_rate * 1000)
+ ret = -EIO;
+ }
+
+ if (WARN_ON(ret)) {
+ if (bLs) {
+ per_cpu(cpu_last_req_freq, cpu) = prev_rate;
+ per_cpu(physical_cluster, cpu) = old_cluster;
+ }
+
+ mutex_unlock(&cluster_lock[new_cluster]);
+
+ return ret;
+ }
+
+ mutex_unlock(&cluster_lock[new_cluster]);
+
+ /* Recalc freq for old cluster when switching clusters */
+ if (old_cluster != new_cluster) {
+ /* Switch cluster */
+ bL_switch_request(cpu, new_cluster);
+
+ mutex_lock(&cluster_lock[old_cluster]);
+
+ /* Set freq of old cluster if there are cpus left on it */
+ new_rate = find_cluster_maxfreq(old_cluster);
+ new_rate = ACTUAL_FREQ(old_cluster, new_rate);
+
+ if (new_rate &&
+ clk_set_rate(clk[old_cluster], new_rate * 1000)) {
+ pr_err("%s: clk_set_rate failed: %d, old cluster: %d\n",
+ __func__, ret, old_cluster);
+ }
+ mutex_unlock(&cluster_lock[old_cluster]);
+ }
+
+ return 0;
+}
+
+/* Set clock frequency */
+static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy,
+ unsigned int index)
+{
+ u32 cpu = policy->cpu, cur_cluster, new_cluster, actual_cluster;
+ unsigned int freqs_new;
+
+ cur_cluster = cpu_to_cluster(cpu);
+ new_cluster = actual_cluster = per_cpu(physical_cluster, cpu);
+
+ freqs_new = freq_table[cur_cluster][index].frequency;
+
+ if (is_bL_switching_enabled()) {
+ if (actual_cluster == A15_CLUSTER && freqs_new < clk_big_min)
+ new_cluster = A7_CLUSTER;
+ else if (actual_cluster == A7_CLUSTER &&
+ freqs_new > clk_little_max)
+ new_cluster = A15_CLUSTER;
+ }
+
+ return ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster,
+ freqs_new);
+}
+
+static inline u32 get_table_count(struct cpufreq_frequency_table *table)
+{
+ int count;
+
+ for (count = 0; table[count].frequency != CPUFREQ_TABLE_END; count++)
+ ;
+
+ return count;
+}
+
+/* get the minimum frequency in the cpufreq_frequency_table */
+static inline u32 get_table_min(struct cpufreq_frequency_table *table)
+{
+ struct cpufreq_frequency_table *pos;
+ u32 min_freq = ~0;
+
+ cpufreq_for_each_entry(pos, table)
+ if (pos->frequency < min_freq)
+ min_freq = pos->frequency;
+ return min_freq;
+}
+
+/* get the maximum frequency in the cpufreq_frequency_table */
+static inline u32 get_table_max(struct cpufreq_frequency_table *table)
+{
+ struct cpufreq_frequency_table *pos;
+ u32 max_freq = 0;
+
+ cpufreq_for_each_entry(pos, table)
+ if (pos->frequency > max_freq)
+ max_freq = pos->frequency;
+ return max_freq;
+}
+
+static bool search_frequency(struct cpufreq_frequency_table *table, int size,
+ unsigned int freq)
+{
+ int count;
+
+ for (count = 0; count < size; count++) {
+ if (table[count].frequency == freq)
+ return true;
+ }
+
+ return false;
+}
+
+static int merge_cluster_tables(void)
+{
+ int i, j, k = 0, count = 1;
+ struct cpufreq_frequency_table *table;
+
+ for (i = 0; i < MAX_CLUSTERS; i++)
+ count += get_table_count(freq_table[i]);
+
+ table = kcalloc(count, sizeof(*table), GFP_KERNEL);
+ if (!table)
+ return -ENOMEM;
+
+ freq_table[MAX_CLUSTERS] = table;
+
+ /* Add in reverse order to get freqs in increasing order */
+ for (i = MAX_CLUSTERS - 1; i >= 0; i--, count = k) {
+ for (j = 0; freq_table[i][j].frequency != CPUFREQ_TABLE_END;
+ j++) {
+ if (i == A15_CLUSTER &&
+ search_frequency(table, count, freq_table[i][j].frequency))
+ continue; /* skip duplicates */
+ table[k++].frequency =
+ VIRT_FREQ(i, freq_table[i][j].frequency);
+ }
+ }
+
+ table[k].driver_data = k;
+ table[k].frequency = CPUFREQ_TABLE_END;
+
+ return 0;
+}
+
+static void _put_cluster_clk_and_freq_table(struct device *cpu_dev,
+ const struct cpumask *cpumask)
+{
+ u32 cluster = raw_cpu_to_cluster(cpu_dev->id);
+
+ if (!freq_table[cluster])
+ return;
+
+ clk_put(clk[cluster]);
+ dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]);
+}
+
+static void put_cluster_clk_and_freq_table(struct device *cpu_dev,
+ const struct cpumask *cpumask)
+{
+ u32 cluster = cpu_to_cluster(cpu_dev->id);
+ int i;
+
+ if (atomic_dec_return(&cluster_usage[cluster]))
+ return;
+
+ if (cluster < MAX_CLUSTERS)
+ return _put_cluster_clk_and_freq_table(cpu_dev, cpumask);
+
+ for_each_present_cpu(i) {
+ struct device *cdev = get_cpu_device(i);
+
+ if (!cdev)
+ return;
+
+ _put_cluster_clk_and_freq_table(cdev, cpumask);
+ }
+
+ /* free virtual table */
+ kfree(freq_table[cluster]);
+}
+
+static int _get_cluster_clk_and_freq_table(struct device *cpu_dev,
+ const struct cpumask *cpumask)
+{
+ u32 cluster = raw_cpu_to_cluster(cpu_dev->id);
+ int ret;
+
+ if (freq_table[cluster])
+ return 0;
+
+ /*
+ * platform specific SPC code must initialise the opp table
+ * so just check if the OPP count is non-zero
+ */
+ ret = dev_pm_opp_get_opp_count(cpu_dev) <= 0;
+ if (ret)
+ goto out;
+
+ ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table[cluster]);
+ if (ret)
+ goto out;
+
+ clk[cluster] = clk_get(cpu_dev, NULL);
+ if (!IS_ERR(clk[cluster]))
+ return 0;
+
+ dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d, cluster: %d\n",
+ __func__, cpu_dev->id, cluster);
+ ret = PTR_ERR(clk[cluster]);
+ dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]);
+
+out:
+ dev_err(cpu_dev, "%s: Failed to get data for cluster: %d\n", __func__,
+ cluster);
+ return ret;
+}
+
+static int get_cluster_clk_and_freq_table(struct device *cpu_dev,
+ const struct cpumask *cpumask)
+{
+ u32 cluster = cpu_to_cluster(cpu_dev->id);
+ int i, ret;
+
+ if (atomic_inc_return(&cluster_usage[cluster]) != 1)
+ return 0;
+
+ if (cluster < MAX_CLUSTERS) {
+ ret = _get_cluster_clk_and_freq_table(cpu_dev, cpumask);
+ if (ret)
+ atomic_dec(&cluster_usage[cluster]);
+ return ret;
+ }
+
+ /*
+ * Get data for all clusters and fill virtual cluster with a merge of
+ * both
+ */
+ for_each_present_cpu(i) {
+ struct device *cdev = get_cpu_device(i);
+
+ if (!cdev)
+ return -ENODEV;
+
+ ret = _get_cluster_clk_and_freq_table(cdev, cpumask);
+ if (ret)
+ goto put_clusters;
+ }
+
+ ret = merge_cluster_tables();
+ if (ret)
+ goto put_clusters;
+
+ /* Assuming 2 cluster, set clk_big_min and clk_little_max */
+ clk_big_min = get_table_min(freq_table[A15_CLUSTER]);
+ clk_little_max = VIRT_FREQ(A7_CLUSTER,
+ get_table_max(freq_table[A7_CLUSTER]));
+
+ return 0;
+
+put_clusters:
+ for_each_present_cpu(i) {
+ struct device *cdev = get_cpu_device(i);
+
+ if (!cdev)
+ return -ENODEV;
+
+ _put_cluster_clk_and_freq_table(cdev, cpumask);
+ }
+
+ atomic_dec(&cluster_usage[cluster]);
+
+ return ret;
+}
+
+/* Per-CPU initialization */
+static int ve_spc_cpufreq_init(struct cpufreq_policy *policy)
+{
+ u32 cur_cluster = cpu_to_cluster(policy->cpu);
+ struct device *cpu_dev;
+ int ret;
+
+ cpu_dev = get_cpu_device(policy->cpu);
+ if (!cpu_dev) {
+ pr_err("%s: failed to get cpu%d device\n", __func__,
+ policy->cpu);
+ return -ENODEV;
+ }
+
+ if (cur_cluster < MAX_CLUSTERS) {
+ int cpu;
+
+ dev_pm_opp_get_sharing_cpus(cpu_dev, policy->cpus);
+
+ for_each_cpu(cpu, policy->cpus)
+ per_cpu(physical_cluster, cpu) = cur_cluster;
+ } else {
+ /* Assumption: during init, we are always running on A15 */
+ per_cpu(physical_cluster, policy->cpu) = A15_CLUSTER;
+ }
+
+ ret = get_cluster_clk_and_freq_table(cpu_dev, policy->cpus);
+ if (ret)
+ return ret;
+
+ policy->freq_table = freq_table[cur_cluster];
+ policy->cpuinfo.transition_latency = 1000000; /* 1 ms */
+
+ if (is_bL_switching_enabled())
+ per_cpu(cpu_last_req_freq, policy->cpu) =
+ clk_get_cpu_rate(policy->cpu);
+
+ dev_info(cpu_dev, "%s: CPU %d initialized\n", __func__, policy->cpu);
+ return 0;
+}
+
+static int ve_spc_cpufreq_exit(struct cpufreq_policy *policy)
+{
+ struct device *cpu_dev;
+
+ cpu_dev = get_cpu_device(policy->cpu);
+ if (!cpu_dev) {
+ pr_err("%s: failed to get cpu%d device\n", __func__,
+ policy->cpu);
+ return -ENODEV;
+ }
+
+ put_cluster_clk_and_freq_table(cpu_dev, policy->related_cpus);
+ return 0;
+}
+
+static struct cpufreq_driver ve_spc_cpufreq_driver = {
+ .name = "vexpress-spc",
+ .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+ CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = ve_spc_cpufreq_set_target,
+ .get = ve_spc_cpufreq_get_rate,
+ .init = ve_spc_cpufreq_init,
+ .exit = ve_spc_cpufreq_exit,
+ .register_em = cpufreq_register_em_with_opp,
+ .attr = cpufreq_generic_attr,
+};
+
+#ifdef CONFIG_BL_SWITCHER
+static int bL_cpufreq_switcher_notifier(struct notifier_block *nfb,
+ unsigned long action, void *_arg)
+{
+ pr_debug("%s: action: %ld\n", __func__, action);
+
+ switch (action) {
+ case BL_NOTIFY_PRE_ENABLE:
+ case BL_NOTIFY_PRE_DISABLE:
+ cpufreq_unregister_driver(&ve_spc_cpufreq_driver);
+ break;
+
+ case BL_NOTIFY_POST_ENABLE:
+ set_switching_enabled(true);
+ cpufreq_register_driver(&ve_spc_cpufreq_driver);
+ break;
+
+ case BL_NOTIFY_POST_DISABLE:
+ set_switching_enabled(false);
+ cpufreq_register_driver(&ve_spc_cpufreq_driver);
+ break;
+
+ default:
+ return NOTIFY_DONE;
+ }
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block bL_switcher_notifier = {
+ .notifier_call = bL_cpufreq_switcher_notifier,
+};
+
+static int __bLs_register_notifier(void)
+{
+ return bL_switcher_register_notifier(&bL_switcher_notifier);
+}
+
+static int __bLs_unregister_notifier(void)
+{
+ return bL_switcher_unregister_notifier(&bL_switcher_notifier);
+}
+#else
+static int __bLs_register_notifier(void) { return 0; }
+static int __bLs_unregister_notifier(void) { return 0; }
+#endif
+
+static int ve_spc_cpufreq_probe(struct platform_device *pdev)
+{
+ int ret, i;
+
+ set_switching_enabled(bL_switcher_get_enabled());
+
+ for (i = 0; i < MAX_CLUSTERS; i++)
+ mutex_init(&cluster_lock[i]);
+
+ if (!is_bL_switching_enabled())
+ ve_spc_cpufreq_driver.flags |= CPUFREQ_IS_COOLING_DEV;
+
+ ret = cpufreq_register_driver(&ve_spc_cpufreq_driver);
+ if (ret) {
+ pr_info("%s: Failed registering platform driver: %s, err: %d\n",
+ __func__, ve_spc_cpufreq_driver.name, ret);
+ } else {
+ ret = __bLs_register_notifier();
+ if (ret)
+ cpufreq_unregister_driver(&ve_spc_cpufreq_driver);
+ else
+ pr_info("%s: Registered platform driver: %s\n",
+ __func__, ve_spc_cpufreq_driver.name);
+ }
+
+ bL_switcher_put_enabled();
+ return ret;
+}
+
+static void ve_spc_cpufreq_remove(struct platform_device *pdev)
+{
+ bL_switcher_get_enabled();
+ __bLs_unregister_notifier();
+ cpufreq_unregister_driver(&ve_spc_cpufreq_driver);
+ bL_switcher_put_enabled();
+ pr_info("%s: Un-registered platform driver: %s\n", __func__,
+ ve_spc_cpufreq_driver.name);
+}
+
+static struct platform_driver ve_spc_cpufreq_platdrv = {
+ .driver = {
+ .name = "vexpress-spc-cpufreq",
+ },
+ .probe = ve_spc_cpufreq_probe,
+ .remove_new = ve_spc_cpufreq_remove,
+};
+module_platform_driver(ve_spc_cpufreq_platdrv);
+
+MODULE_ALIAS("platform:vexpress-spc-cpufreq");
+MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>");
+MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>");
+MODULE_DESCRIPTION("Vexpress SPC ARM big LITTLE cpufreq driver");
+MODULE_LICENSE("GPL v2");