proj/linux-patches:6.12 commit in: /
"Mike Pagano" <[email protected]> Mon, 03 Aug 2026 11:19:00 +0000 (UTC)
| Newsgroups | gmane.linux.gentoo.cvs |
|---|---|
| Message-ID | <1785755927.522b525443998a886e956e8648365307ab77f7cd.mpagano@gentoo> |
commit: 522b525443998a886e956e8648365307ab77f7cd
Author: Mike Pagano <mpagano <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 3 11:18:47 2026 +0000
Commit: Mike Pagano <mpagano <AT> gentoo <DOT> org>
CommitDate: Mon Aug 3 11:18:47 2026 +0000
URL: https://gitweb.gentoo.org/proj/linux-patches.git/commit/?id=522b5254
Linux patch 6.12.101
Signed-off-by: Mike Pagano <mpagano <AT> gentoo.org>
0000_README | 4 +
1100_linux-6.12.101.patch | 47122 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 47126 insertions(+)
diff --git a/0000_README b/0000_README
index 9049269c..1db0b38b 100644
--- a/0000_README
+++ b/0000_README
@@ -443,6 +443,10 @@ Patch: 1099_linux-6.12.100.patch
From: https://www.kernel.org
Desc: Linux 6.12.100
+Patch: 1100_linux-6.12.101.patch
+From: https://www.kernel.org
+Desc: Linux 6.12.101
+
Patch: 1500_fortify-copy-size-value-range-tracking-fix.patch
From: https://git.kernel.org/
Desc: fortify: Hide run-time copy size from value range tracking
diff --git a/1100_linux-6.12.101.patch b/1100_linux-6.12.101.patch
new file mode 100644
index 00000000..a24d4642
--- /dev/null
+++ b/1100_linux-6.12.101.patch
@@ -0,0 +1,47122 @@
+diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst
+index d55751cad67cf9..8e0d31230b29b5 100644
+--- a/Documentation/gpu/drm-mm.rst
++++ b/Documentation/gpu/drm-mm.rst
+@@ -509,8 +509,14 @@ DRM GPUVM Function References
+ DRM Buddy Allocator
+ ===================
+
+-DRM Buddy Function References
+------------------------------
++Buddy Allocator Function References (GPU buddy)
++-----------------------------------------------
++
++.. kernel-doc:: drivers/gpu/buddy.c
++ :export:
++
++DRM Buddy Specific Logging Function References
++----------------------------------------------
+
+ .. kernel-doc:: drivers/gpu/drm/drm_buddy.c
+ :export:
+diff --git a/Documentation/netlink/specs/dpll.yaml b/Documentation/netlink/specs/dpll.yaml
+index cb5763f79ac2ae..c799d43b8833d1 100644
+--- a/Documentation/netlink/specs/dpll.yaml
++++ b/Documentation/netlink/specs/dpll.yaml
+@@ -85,6 +85,36 @@ definitions:
+ This may happen for example if dpll device was previously
+ locked on an input pin of type PIN_TYPE_SYNCE_ETH_PORT.
+ render-max: true
++ -
++ type: enum
++ name: clock-quality-level
++ doc: |
++ level of quality of a clock device. This mainly applies when
++ the dpll lock-status is DPLL_LOCK_STATUS_HOLDOVER.
++ The current list is defined according to the table 11-7 contained
++ in ITU-T G.8264/Y.1364 document. One may extend this list freely
++ by other ITU-T defined clock qualities, or different ones defined
++ by another standardization body (for those, please use
++ different prefix).
++ entries:
++ -
++ name: itu-opt1-prc
++ value: 1
++ -
++ name: itu-opt1-ssu-a
++ -
++ name: itu-opt1-ssu-b
++ -
++ name: itu-opt1-eec1
++ -
++ name: itu-opt1-prtc
++ -
++ name: itu-opt1-eprtc
++ -
++ name: itu-opt1-eeec
++ -
++ name: itu-opt1-eprc
++ render-max: true
+ -
+ type: const
+ name: temp-divider
+@@ -252,6 +282,17 @@ attribute-sets:
+ name: lock-status-error
+ type: u32
+ enum: lock-status-error
++ -
++ name: clock-quality-level
++ type: u32
++ enum: clock-quality-level
++ multi-attr: true
++ doc: |
++ Level of quality of a clock device. This mainly applies when
++ the dpll lock-status is DPLL_LOCK_STATUS_HOLDOVER. This could
++ be put to message multiple times to indicate possible parallel
++ quality levels (e.g. one specified by ITU option 1 and another
++ one specified by option 2).
+ -
+ name: pin
+ enum-name: dpll_a_pin
+diff --git a/Documentation/netlink/specs/rt_link.yaml b/Documentation/netlink/specs/rt_link.yaml
+index a048fc30389d68..5fb757dcb53c2e 100644
+--- a/Documentation/netlink/specs/rt_link.yaml
++++ b/Documentation/netlink/specs/rt_link.yaml
+@@ -1619,31 +1619,31 @@ attribute-sets:
+ type: u32
+ -
+ name: mode
+- type: flag
++ type: u8
+ -
+ name: guard
+- type: flag
++ type: u8
+ -
+ name: protect
+- type: flag
++ type: u8
+ -
+ name: fast-leave
+- type: flag
++ type: u8
+ -
+ name: learning
+- type: flag
++ type: u8
+ -
+ name: unicast-flood
+- type: flag
++ type: u8
+ -
+ name: proxyarp
+- type: flag
++ type: u8
+ -
+ name: learning-sync
+- type: flag
++ type: u8
+ -
+ name: proxyarp-wifi
+- type: flag
++ type: u8
+ -
+ name: root-id
+ type: binary
+@@ -1690,34 +1690,34 @@ attribute-sets:
+ type: pad
+ -
+ name: mcast-flood
+- type: flag
++ type: u8
+ -
+ name: mcast-to-ucast
+- type: flag
++ type: u8
+ -
+ name: vlan-tunnel
+- type: flag
++ type: u8
+ -
+ name: bcast-flood
+- type: flag
++ type: u8
+ -
+ name: group-fwd-mask
+ type: u16
+ -
+ name: neigh-suppress
+- type: flag
++ type: u8
+ -
+ name: isolated
+- type: flag
++ type: u8
+ -
+ name: backup-port
+ type: u32
+ -
+ name: mrp-ring-open
+- type: flag
++ type: u8
+ -
+ name: mrp-in-open
+- type: flag
++ type: u8
+ -
+ name: mcast-eht-hosts-limit
+ type: u32
+@@ -1726,10 +1726,10 @@ attribute-sets:
+ type: u32
+ -
+ name: locked
+- type: flag
++ type: u8
+ -
+ name: mab
+- type: flag
++ type: u8
+ -
+ name: mcast-n-groups
+ type: u32
+@@ -1738,7 +1738,7 @@ attribute-sets:
+ type: u32
+ -
+ name: neigh-vlan-suppress
+- type: flag
++ type: u8
+ -
+ name: backup-nhid
+ type: u32
+diff --git a/Makefile b/Makefile
+index 40f2b39cfced6d..45c45c42694dc2 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,7 +1,7 @@
+ # SPDX-License-Identifier: GPL-2.0
+ VERSION = 6
+ PATCHLEVEL = 12
+-SUBLEVEL = 100
++SUBLEVEL = 101
+ EXTRAVERSION =
+ NAME = Baby Opossum Posse
+
+@@ -443,6 +443,10 @@ KBUILD_USERLDFLAGS := $(USERLDFLAGS)
+
+ # These flags apply to all Rust code in the tree, including the kernel and
+ # host programs.
++#
++# `-Aclippy::unwrap_or_default`: the lint is buggy [1] and ignores our
++# MSRV. It can trigger depending on the optimization level.
++# [1] https://github.com/rust-lang/rust-clippy/issues/17379
+ export rust_common_flags := --edition=2021 \
+ -Zbinary_dep_depinfo=y \
+ -Astable_features \
+@@ -464,6 +468,7 @@ export rust_common_flags := --edition=2021 \
+ -Aclippy::uninlined_format_args \
+ -Wclippy::unnecessary_safety_comment \
+ -Wclippy::unnecessary_safety_doc \
++ -Aclippy::unwrap_or_default \
+ -Wrustdoc::missing_crate_level_docs \
+ -Wrustdoc::unescaped_backticks
+
+diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
+index 75428e70a69252..c1d2d91975057a 100644
+--- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi
++++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
+@@ -5262,7 +5262,7 @@
+ #size-cells = <0>;
+
+ cpu0_0: cpu@0 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x00000>;
+
+@@ -5281,7 +5281,7 @@
+ };
+
+ cpu0_1: cpu@100 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x00100>;
+
+@@ -5300,7 +5300,7 @@
+ };
+
+ cpu0_2: cpu@200 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x00200>;
+
+@@ -5319,7 +5319,7 @@
+ };
+
+ cpu0_3: cpu@300 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x00300>;
+
+@@ -5338,7 +5338,7 @@
+ };
+
+ cpu1_0: cpu@10000 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x10000>;
+
+@@ -5357,7 +5357,7 @@
+ };
+
+ cpu1_1: cpu@10100 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x10100>;
+
+@@ -5376,7 +5376,7 @@
+ };
+
+ cpu1_2: cpu@10200 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x10200>;
+
+@@ -5395,7 +5395,7 @@
+ };
+
+ cpu1_3: cpu@10300 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x10300>;
+
+@@ -5414,7 +5414,7 @@
+ };
+
+ cpu2_0: cpu@20000 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x20000>;
+
+@@ -5433,7 +5433,7 @@
+ };
+
+ cpu2_1: cpu@20100 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x20100>;
+
+@@ -5452,7 +5452,7 @@
+ };
+
+ cpu2_2: cpu@20200 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x20200>;
+
+@@ -5471,7 +5471,7 @@
+ };
+
+ cpu2_3: cpu@20300 {
+- compatible = "arm,cortex-a78";
++ compatible = "arm,cortex-a78ae";
+ device_type = "cpu";
+ reg = <0x20300>;
+
+diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi
+index e52d938df1bece..d8fb0d2a89caf2 100644
+--- a/arch/arm64/boot/dts/qcom/sc7180.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi
+@@ -3410,8 +3410,8 @@
+ dp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi
+index b47ad94f7d1a21..088dc46dd32ba9 100644
+--- a/arch/arm64/boot/dts/qcom/sc7280.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi
+@@ -4723,8 +4723,8 @@
+ edp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -4820,8 +4820,8 @@
+ dp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
+index 6ca9fba74c08ba..4715e26fc26479 100644
+--- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
+@@ -3269,8 +3269,8 @@
+ dp0_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -3347,8 +3347,8 @@
+ dp1_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -3415,8 +3415,8 @@
+ edp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+index c10ee18cb611ad..3fa2e13f4baf02 100644
+--- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
++++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+@@ -4259,8 +4259,8 @@
+ mdss0_dp0_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -4337,8 +4337,8 @@
+ mdss0_dp1_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -4409,8 +4409,8 @@
+ mdss0_dp2_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -4481,8 +4481,8 @@
+ mdss0_dp3_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -5547,8 +5547,8 @@
+ mdss1_dp0_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -5619,8 +5619,8 @@
+ mdss1_dp1_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -5691,8 +5691,8 @@
+ mdss1_dp2_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+@@ -5763,8 +5763,8 @@
+ mdss1_dp3_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi
+index 10418fccfea24f..1234df16d51f28 100644
+--- a/arch/arm64/boot/dts/qcom/sm6350.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi
+@@ -2222,8 +2222,8 @@
+ dp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi
+index 27f87835bc5595..0aeceffd872c09 100644
+--- a/arch/arm64/boot/dts/qcom/sm8150.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi
+@@ -3932,8 +3932,8 @@
+ dp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi
+index 9995fc515e113d..4cb91f31340dbd 100644
+--- a/arch/arm64/boot/dts/qcom/sm8250.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi
+@@ -4818,8 +4818,8 @@
+ dp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi
+index 0be8f2befec7c5..1870ec4176daa3 100644
+--- a/arch/arm64/boot/dts/qcom/sm8350.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi
+@@ -2909,8 +2909,8 @@
+ dp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi
+index 4147d3164574d3..59172db1e3d9e2 100644
+--- a/arch/arm64/boot/dts/qcom/sm8450.dtsi
++++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi
+@@ -3223,8 +3223,8 @@
+ dp_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
++ opp-162000000 {
++ opp-hz = /bits/ 64 <162000000>;
+ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+diff --git a/arch/arm64/boot/dts/qcom/x1e80100.dtsi b/arch/arm64/boot/dts/qcom/x1e80100.dtsi
+index 8843dac449000c..7e9de1b7c7a314 100644
+--- a/arch/arm64/boot/dts/qcom/x1e80100.dtsi
++++ b/arch/arm64/boot/dts/qcom/x1e80100.dtsi
+@@ -4758,18 +4758,18 @@
+ mdss_dp0_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
+- required-opps = <&rpmhpd_opp_low_svs>;
+- };
+-
+ opp-270000000 {
+ opp-hz = /bits/ 64 <270000000>;
+- required-opps = <&rpmhpd_opp_svs>;
++ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+ opp-540000000 {
+ opp-hz = /bits/ 64 <540000000>;
++ required-opps = <&rpmhpd_opp_svs>;
++ };
++
++ opp-594000000 {
++ opp-hz = /bits/ 64 <594000000>;
+ required-opps = <&rpmhpd_opp_svs_l1>;
+ };
+
+@@ -4806,7 +4806,7 @@
+ assigned-clock-parents = <&usb_1_ss1_qmpphy QMP_USB43DP_DP_LINK_CLK>,
+ <&usb_1_ss1_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>;
+
+- operating-points-v2 = <&mdss_dp1_opp_table>;
++ operating-points-v2 = <&mdss_dp0_opp_table>;
+
+ power-domains = <&rpmhpd RPMHPD_MMCX>;
+
+@@ -4837,30 +4837,6 @@
+ };
+ };
+ };
+-
+- mdss_dp1_opp_table: opp-table {
+- compatible = "operating-points-v2";
+-
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
+- required-opps = <&rpmhpd_opp_low_svs>;
+- };
+-
+- opp-270000000 {
+- opp-hz = /bits/ 64 <270000000>;
+- required-opps = <&rpmhpd_opp_svs>;
+- };
+-
+- opp-540000000 {
+- opp-hz = /bits/ 64 <540000000>;
+- required-opps = <&rpmhpd_opp_svs_l1>;
+- };
+-
+- opp-810000000 {
+- opp-hz = /bits/ 64 <810000000>;
+- required-opps = <&rpmhpd_opp_nom>;
+- };
+- };
+ };
+
+ mdss_dp2: displayport-controller@ae9a000 {
+@@ -4889,7 +4865,7 @@
+ assigned-clock-parents = <&usb_1_ss2_qmpphy QMP_USB43DP_DP_LINK_CLK>,
+ <&usb_1_ss2_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>;
+
+- operating-points-v2 = <&mdss_dp2_opp_table>;
++ operating-points-v2 = <&mdss_dp0_opp_table>;
+
+ power-domains = <&rpmhpd RPMHPD_MMCX>;
+
+@@ -4919,30 +4895,6 @@
+ };
+ };
+ };
+-
+- mdss_dp2_opp_table: opp-table {
+- compatible = "operating-points-v2";
+-
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
+- required-opps = <&rpmhpd_opp_low_svs>;
+- };
+-
+- opp-270000000 {
+- opp-hz = /bits/ 64 <270000000>;
+- required-opps = <&rpmhpd_opp_svs>;
+- };
+-
+- opp-540000000 {
+- opp-hz = /bits/ 64 <540000000>;
+- required-opps = <&rpmhpd_opp_svs_l1>;
+- };
+-
+- opp-810000000 {
+- opp-hz = /bits/ 64 <810000000>;
+- required-opps = <&rpmhpd_opp_nom>;
+- };
+- };
+ };
+
+ mdss_dp3: displayport-controller@aea0000 {
+@@ -5002,19 +4954,14 @@
+ mdss_dp3_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+- opp-160000000 {
+- opp-hz = /bits/ 64 <160000000>;
+- required-opps = <&rpmhpd_opp_low_svs>;
+- };
+-
+ opp-270000000 {
+ opp-hz = /bits/ 64 <270000000>;
+- required-opps = <&rpmhpd_opp_svs>;
++ required-opps = <&rpmhpd_opp_low_svs>;
+ };
+
+- opp-540000000 {
+- opp-hz = /bits/ 64 <540000000>;
+- required-opps = <&rpmhpd_opp_svs_l1>;
++ opp-594000000 {
++ opp-hz = /bits/ 64 <594000000>;
++ required-opps = <&rpmhpd_opp_svs>;
+ };
+
+ opp-810000000 {
+diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
+index f94378379447a6..0b6e371eea98d2 100644
+--- a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
++++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
+@@ -134,7 +134,6 @@
+ gpios = <&main_gpio0 31 GPIO_ACTIVE_HIGH>;
+ states = <1800000 0x0>,
+ <3300000 0x1>;
+- bootph-all;
+ };
+
+ leds {
+@@ -382,7 +381,6 @@
+ pinctrl-single,pins = <
+ AM62AX_MCU_IOPAD(0x000, PIN_INPUT, 7) /* (E11) MCU_GPIO0_0 */
+ >;
+- bootph-all;
+ };
+ };
+
+diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
+index 2b5e0ccc0af555..c29d16284fe819 100644
+--- a/arch/arm64/mm/mmu.c
++++ b/arch/arm64/mm/mmu.c
+@@ -1384,12 +1384,13 @@ int arch_add_memory(int nid, u64 start, u64 size,
+ return ret;
+ }
+
+-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
++void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
++ struct dev_pagemap *pgmap)
+ {
+ unsigned long start_pfn = start >> PAGE_SHIFT;
+ unsigned long nr_pages = size >> PAGE_SHIFT;
+
+- __remove_pages(start_pfn, nr_pages, altmap);
++ __remove_pages(start_pfn, nr_pages, altmap, pgmap);
+ __remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
+ }
+
+diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c
+index 1e2bfe5d1485f6..df9c0f8058d87b 100644
+--- a/arch/loongarch/kernel/acpi.c
++++ b/arch/loongarch/kernel/acpi.c
+@@ -200,10 +200,12 @@ static void __init acpi_process_madt(void)
+ }
+
+ int pptt_enabled;
++static int acpi_nr_packages;
++static int acpi_package_ids[MAX_PACKAGES];
+
+ int __init parse_acpi_topology(void)
+ {
+- int cpu, topology_id;
++ int i, cpu, topology_id;
+
+ for_each_possible_cpu(cpu) {
+ topology_id = find_acpi_cpu_topology(cpu, 0);
+@@ -221,6 +223,29 @@ int __init parse_acpi_topology(void)
+
+ cpu_data[cpu].core = topology_id;
+ }
++
++ topology_id = find_acpi_cpu_topology_package(cpu);
++ if (topology_id < 0) {
++ pr_warn("Invalid BIOS PPTT\n");
++ return -ENOENT;
++ }
++
++ for (i = 0; i < acpi_nr_packages; i++)
++ if (acpi_package_ids[i] == topology_id)
++ break;
++
++ if (i == acpi_nr_packages)
++ acpi_package_ids[acpi_nr_packages++] = topology_id;
++
++ cpu_data[cpu].package = topology_id;
++ }
++
++ for_each_possible_cpu(cpu) {
++ for (i = 0; i < acpi_nr_packages; i++)
++ if (cpu_data[cpu].package == acpi_package_ids[i]) {
++ cpu_data[cpu].package = i; /* Canonicalize */
++ break;
++ }
+ }
+
+ pptt_enabled = 1;
+diff --git a/arch/loongarch/kernel/kgdb.c b/arch/loongarch/kernel/kgdb.c
+index 7be5b4c0c90020..6728b2d67dd325 100644
+--- a/arch/loongarch/kernel/kgdb.c
++++ b/arch/loongarch/kernel/kgdb.c
+@@ -252,7 +252,8 @@ static int kgdb_loongarch_notify(struct notifier_block *self, unsigned long cmd,
+ if (atomic_read(&kgdb_active) != -1)
+ kgdb_nmicallback(smp_processor_id(), regs);
+
+- if (kgdb_handle_exception(args->trapnr, args->signr, cmd, regs))
++ if (kgdb_handle_exception(regs->csr_era == stepped_address ? 0 : args->trapnr,
++ args->signr, cmd, regs))
+ return NOTIFY_DONE;
+
+ if (atomic_read(&kgdb_setting_breakpoint))
+diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
+index 1e482edcf46c45..4a6279886e67d2 100644
+--- a/arch/loongarch/kernel/setup.c
++++ b/arch/loongarch/kernel/setup.c
+@@ -610,6 +610,7 @@ void __init setup_arch(char **cmdline_p)
+ memblock_init();
+ pagetable_init();
+ bootcmdline_init(cmdline_p);
++ jump_label_init(); /* Initialise the static keys for early params */
+ parse_early_param();
+ reserve_initrd_mem();
+
+@@ -617,8 +618,6 @@ void __init setup_arch(char **cmdline_p)
+ arch_mem_init(cmdline_p);
+
+ resource_init();
+- jump_label_init(); /* Initialise the static keys for paravirtualization */
+-
+ #ifdef CONFIG_SMP
+ plat_smp_setup();
+ prefill_possible_map();
+diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c
+index cf1f377ec622e8..780abcd86ca327 100644
+--- a/arch/loongarch/kernel/smp.c
++++ b/arch/loongarch/kernel/smp.c
+@@ -377,10 +377,10 @@ void loongson_init_secondary(void)
+ numa_add_cpu(cpu);
+ #endif
+ per_cpu(cpu_state, cpu) = CPU_ONLINE;
+- cpu_data[cpu].package =
+- cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
+ cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core :
+ cpu_logical_map(cpu) % loongson_sysconf.cores_per_package;
++ cpu_data[cpu].package = pptt_enabled ? cpu_data[cpu].package :
++ cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
+ cpu_data[cpu].global_id = cpu_logical_map(cpu);
+ }
+
+diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
+index c595041afed4d8..a6c5fd2df2f6de 100644
+--- a/arch/loongarch/mm/init.c
++++ b/arch/loongarch/mm/init.c
+@@ -103,12 +103,13 @@ int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params)
+ return ret;
+ }
+
+-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
++void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
++ struct dev_pagemap *pgmap)
+ {
+ unsigned long start_pfn = start >> PAGE_SHIFT;
+ unsigned long nr_pages = size >> PAGE_SHIFT;
+
+- __remove_pages(start_pfn, nr_pages, altmap);
++ __remove_pages(start_pfn, nr_pages, altmap, pgmap);
+ }
+
+ #ifdef CONFIG_NUMA
+diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
+index ce7f91172ec2b6..c69c0f23f5212a 100644
+--- a/arch/powerpc/kernel/time.c
++++ b/arch/powerpc/kernel/time.c
+@@ -376,6 +376,47 @@ void vtime_task_switch(struct task_struct *prev)
+ acct->starttime = acct0->starttime;
+ }
+ }
++
++/**
++ * vtime_reset - Fast forward vtime entry clocks
++ *
++ * Called from dynticks idle IRQ entry to fast-forward the clocks to current time
++ * so that the IRQ time is still accounted by vtime while nohz cputime is paused.
++ */
++void vtime_reset(void)
++{
++ struct cpu_accounting_data *acct = get_accounting(current);
++
++ acct->starttime = mftb();
++#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
++ acct->startspurr = read_spurr(acct->starttime);
++#endif
++}
++
++#ifdef CONFIG_NO_HZ_COMMON
++/**
++ * vtime_dyntick_start - Inform vtime about entry to idle-dynticks
++ *
++ * Called when idle enters in dyntick mode. The idle cputime that elapsed so far
++ * is accumulated and the tick subsystem takes over the idle cputime accounting.
++ */
++void vtime_dyntick_start(void)
++{
++ vtime_account_idle(current);
++}
++
++/**
++ * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks
++ *
++ * Called when idle exits from dyntick mode. The vtime entry clocks are
++ * fast-forward to current time so that idle accounting restarts elapsing from
++ * now.
++ */
++void vtime_dyntick_stop(void)
++{
++ vtime_reset();
++}
++#endif /* CONFIG_NO_HZ_COMMON */
+ #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
+
+ void __no_kcsan __delay(unsigned long loops)
+@@ -886,6 +927,7 @@ static void __init set_decrementer_max(void)
+ static void __init init_decrementer_clockevent(void)
+ {
+ register_decrementer_clockevent(smp_processor_id());
++ vtime_reset();
+ }
+
+ void secondary_cpu_time_init(void)
+@@ -901,6 +943,7 @@ void secondary_cpu_time_init(void)
+ /* FIME: Should make unrelated change to move snapshot_timebase
+ * call here ! */
+ register_decrementer_clockevent(smp_processor_id());
++ vtime_reset();
+ }
+
+ /* This function is only called on the boot processor */
+diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
+index 1221c561b43a0b..7ddecbd9675198 100644
+--- a/arch/powerpc/mm/mem.c
++++ b/arch/powerpc/mm/mem.c
+@@ -157,12 +157,13 @@ int __ref arch_add_memory(int nid, u64 start, u64 size,
+ return rc;
+ }
+
+-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
++void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
++ struct dev_pagemap *pgmap)
+ {
+ unsigned long start_pfn = start >> PAGE_SHIFT;
+ unsigned long nr_pages = size >> PAGE_SHIFT;
+
+- __remove_pages(start_pfn, nr_pages, altmap);
++ __remove_pages(start_pfn, nr_pages, altmap, pgmap);
+ arch_remove_linear_mapping(start, size);
+ }
+ #endif
+diff --git a/arch/powerpc/platforms/85xx/common.c b/arch/powerpc/platforms/85xx/common.c
+index 757811155587db..c11deb2f50ed45 100644
+--- a/arch/powerpc/platforms/85xx/common.c
++++ b/arch/powerpc/platforms/85xx/common.c
+@@ -42,6 +42,8 @@ static const struct of_device_id mpc85xx_common_ids[] __initconst = {
+ { .compatible = "fsl,qoriq-pcie-v2.3", },
+ { .compatible = "fsl,qoriq-pcie-v2.2", },
+ { .compatible = "fsl,fman", },
++ /* IFC NAND and NOR controllers */
++ { .compatible = "fsl,ifc", },
+ {},
+ };
+
+diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
+index 2e2254fd2a2aae..7beef600abad7c 100644
+--- a/arch/riscv/include/asm/kvm_host.h
++++ b/arch/riscv/include/asm/kvm_host.h
+@@ -232,13 +232,13 @@ struct kvm_vcpu_arch {
+ /*
+ * VCPU interrupts
+ *
+- * We have a lockless approach for tracking pending VCPU interrupts
+- * implemented using atomic bitops. The irqs_pending bitmap represent
+- * pending interrupts whereas irqs_pending_mask represent bits changed
+- * in irqs_pending. Our approach is modeled around multiple producer
+- * and single consumer problem where the consumer is the VCPU itself.
++ * The irqs_pending bitmap represents pending interrupts whereas
++ * irqs_pending_mask represents bits changed in irqs_pending. Updates
++ * to these bitmaps are serialized so vcpu interrupt sync/flush cannot
++ * drop a newly injected interrupt while syncing guest-visible HVIP.
+ */
+ #define KVM_RISCV_VCPU_NR_IRQS 64
++ raw_spinlock_t irqs_pending_lock;
+ DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
+ DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);
+
+diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c
+index fc62548888c587..d273080bec34d8 100644
+--- a/arch/riscv/kernel/sys_hwprobe.c
++++ b/arch/riscv/kernel/sys_hwprobe.c
+@@ -336,6 +336,7 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
+ if (cpusetsize > cpumask_size())
+ cpusetsize = cpumask_size();
+
++ cpumask_clear(&cpus);
+ ret = copy_from_user(&cpus, cpus_user, cpusetsize);
+ if (ret)
+ return -EFAULT;
+diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c
+index 9f3b527596ded8..01319d8d75b0a4 100644
+--- a/arch/riscv/kvm/aia.c
++++ b/arch/riscv/kvm/aia.c
+@@ -71,12 +71,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu)
+ struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
+ unsigned long mask, val;
+
++ lockdep_assert_held(&vcpu->arch.irqs_pending_lock);
++
+ if (!kvm_riscv_aia_available())
+ return;
+
+- if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) {
+- mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0);
+- val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask;
++ mask = vcpu->arch.irqs_pending_mask[1];
++ if (mask) {
++ vcpu->arch.irqs_pending_mask[1] = 0;
++ val = vcpu->arch.irqs_pending[1] & mask;
+
+ csr->hviph &= ~mask;
+ csr->hviph |= val;
+@@ -87,6 +90,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
+
++ lockdep_assert_held(&vcpu->arch.irqs_pending_lock);
++
+ if (kvm_riscv_aia_available())
+ csr->vsieh = csr_read(CSR_VSIEH);
+ }
+@@ -96,13 +101,21 @@ bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
+ {
+ int hgei;
+ unsigned long seip;
++#ifdef CONFIG_32BIT
++ unsigned long flags;
++ bool pending;
++#endif
+
+ if (!kvm_riscv_aia_available())
+ return false;
+
+ #ifdef CONFIG_32BIT
+- if (READ_ONCE(vcpu->arch.irqs_pending[1]) &
+- (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask)))
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
++ pending = vcpu->arch.irqs_pending[1] &
++ (vcpu->arch.aia_context.guest_csr.vsieh &
++ upper_32_bits(mask));
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
++ if (pending)
+ return true;
+ #endif
+
+@@ -190,6 +203,9 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
+ unsigned long val)
+ {
+ struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
++#ifdef CONFIG_32BIT
++ unsigned long flags;
++#endif
+
+ if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
+ return -ENOENT;
+@@ -198,8 +214,12 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
+ ((unsigned long *)csr)[reg_num] = val;
+
+ #ifdef CONFIG_32BIT
+- if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph))
+- WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0);
++ if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) {
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
++ vcpu->arch.irqs_pending_mask[1] = 0;
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock,
++ flags);
++ }
+ #endif
+ }
+
+diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
+index da01c81bf41676..ad519c9c31d05a 100644
+--- a/arch/riscv/kvm/vcpu.c
++++ b/arch/riscv/kvm/vcpu.c
+@@ -52,6 +52,7 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)
+ struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
+ struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
+ struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context;
++ unsigned long flags;
+ bool loaded;
+
+ /**
+@@ -80,8 +81,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)
+
+ kvm_riscv_vcpu_aia_reset(vcpu);
+
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+ bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
+ bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
+
+ kvm_riscv_vcpu_pmu_reset(vcpu);
+
+@@ -126,6 +129,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
+ /* Setup VCPU hfence queue */
+ spin_lock_init(&vcpu->arch.hfence_lock);
+
++ raw_spin_lock_init(&vcpu->arch.irqs_pending_lock);
++
+ /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */
+ spin_lock_init(&vcpu->arch.reset_cntx_lock);
+
+@@ -341,10 +346,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
+ unsigned long mask, val;
++ unsigned long flags;
++
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+
+- if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) {
+- mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0);
+- val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask;
++ mask = vcpu->arch.irqs_pending_mask[0];
++ if (mask) {
++ vcpu->arch.irqs_pending_mask[0] = 0;
++ val = vcpu->arch.irqs_pending[0] & mask;
+
+ csr->hvip &= ~mask;
+ csr->hvip |= val;
+@@ -352,11 +361,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
+
+ /* Flush AIA high interrupts */
+ kvm_riscv_vcpu_aia_flush_interrupts(vcpu);
++
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
+ }
+
+ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
+ {
+ unsigned long hvip;
++ unsigned long flags;
+ struct kvm_vcpu_arch *v = &vcpu->arch;
+ struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
+
+@@ -365,34 +377,41 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
+
+ /* Sync-up HVIP.VSSIP bit changes does by Guest */
+ hvip = csr_read(CSR_HVIP);
++
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
++
+ if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
+ if (hvip & (1UL << IRQ_VS_SOFT)) {
+- if (!test_and_set_bit(IRQ_VS_SOFT,
+- v->irqs_pending_mask))
+- set_bit(IRQ_VS_SOFT, v->irqs_pending);
++ if (!__test_and_set_bit(IRQ_VS_SOFT,
++ v->irqs_pending_mask))
++ __set_bit(IRQ_VS_SOFT, v->irqs_pending);
+ } else {
+- if (!test_and_set_bit(IRQ_VS_SOFT,
+- v->irqs_pending_mask))
+- clear_bit(IRQ_VS_SOFT, v->irqs_pending);
++ if (!__test_and_set_bit(IRQ_VS_SOFT,
++ v->irqs_pending_mask))
++ __clear_bit(IRQ_VS_SOFT, v->irqs_pending);
+ }
+ }
+
+ /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */
+ if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) {
+ if (!(hvip & (1UL << IRQ_PMU_OVF)) &&
+- !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
+- clear_bit(IRQ_PMU_OVF, v->irqs_pending);
++ !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
++ __clear_bit(IRQ_PMU_OVF, v->irqs_pending);
+ }
+
+ /* Sync-up AIA high interrupts */
+ kvm_riscv_vcpu_aia_sync_interrupts(vcpu);
+
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
++
+ /* Sync-up timer CSRs */
+ kvm_riscv_vcpu_timer_sync(vcpu);
+ }
+
+ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
+ {
++ unsigned long flags;
++
+ /*
+ * We only allow VS-mode software, timer, and external
+ * interrupts when irq is one of the local interrupts
+@@ -405,9 +424,10 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
+ irq != IRQ_PMU_OVF)
+ return -EINVAL;
+
+- set_bit(irq, vcpu->arch.irqs_pending);
+- smp_mb__before_atomic();
+- set_bit(irq, vcpu->arch.irqs_pending_mask);
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
++ __set_bit(irq, vcpu->arch.irqs_pending);
++ __set_bit(irq, vcpu->arch.irqs_pending_mask);
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
+
+ kvm_vcpu_kick(vcpu);
+
+@@ -416,6 +436,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
+
+ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
+ {
++ unsigned long flags;
++
+ /*
+ * We only allow VS-mode software, timer, counter overflow and external
+ * interrupts when irq is one of the local interrupts
+@@ -428,26 +450,33 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
+ irq != IRQ_PMU_OVF)
+ return -EINVAL;
+
+- clear_bit(irq, vcpu->arch.irqs_pending);
+- smp_mb__before_atomic();
+- set_bit(irq, vcpu->arch.irqs_pending_mask);
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
++ __clear_bit(irq, vcpu->arch.irqs_pending);
++ __set_bit(irq, vcpu->arch.irqs_pending_mask);
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
+
+ return 0;
+ }
+
+ bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
+ {
++ unsigned long flags;
+ unsigned long ie;
++ bool ret;
+
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+ ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
+ << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask;
+ ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &
+ (unsigned long)mask;
+- if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie)
+- return true;
++ ret = vcpu->arch.irqs_pending[0] & ie;
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
+
+ /* Check AIA high interrupts */
+- return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
++ if (!ret)
++ ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
++
++ return ret;
+ }
+
+ void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
+diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c
+index b319c4c13c54ce..6223806fcbc2ba 100644
+--- a/arch/riscv/kvm/vcpu_onereg.c
++++ b/arch/riscv/kvm/vcpu_onereg.c
+@@ -445,6 +445,7 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
+ unsigned long reg_val)
+ {
+ struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
++ unsigned long flags;
+
+ if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
+ return -ENOENT;
+@@ -456,8 +457,11 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
+
+ ((unsigned long *)csr)[reg_num] = reg_val;
+
+- if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
+- WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0);
++ if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
++ raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
++ vcpu->arch.irqs_pending_mask[0] = 0;
++ raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
++ }
+
+ return 0;
+ }
+diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
+index 8cd8bc9b82cb29..4594f77a06ee3d 100644
+--- a/arch/riscv/mm/init.c
++++ b/arch/riscv/mm/init.c
+@@ -1839,9 +1839,10 @@ int __ref arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *param
+ return ret;
+ }
+
+-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
++void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
++ struct dev_pagemap *pgmap)
+ {
+- __remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap);
++ __remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap, pgmap);
+ remove_linear_mapping(start, size);
+ flush_tlb_all();
+ }
+diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c
+index 458abd9bac7025..9d74ceff136c54 100644
+--- a/arch/s390/lib/csum-partial.c
++++ b/arch/s390/lib/csum-partial.c
+@@ -23,7 +23,7 @@ static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __w
+ if (!cpu_has_vx()) {
+ if (copy)
+ memcpy(dst, src, len);
+- return cksm(dst, len, sum);
++ return cksm(src, len, sum);
+ }
+ kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23);
+ fpu_vlvgf(16, (__force u32)sum, 1);
+diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
+index 7a96623a9d2e43..58f535fd9a334d 100644
+--- a/arch/s390/mm/init.c
++++ b/arch/s390/mm/init.c
+@@ -290,12 +290,13 @@ int arch_add_memory(int nid, u64 start, u64 size,
+ return rc;
+ }
+
+-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
++void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
++ struct dev_pagemap *pgmap)
+ {
+ unsigned long start_pfn = start >> PAGE_SHIFT;
+ unsigned long nr_pages = size >> PAGE_SHIFT;
+
+- __remove_pages(start_pfn, nr_pages, altmap);
++ __remove_pages(start_pfn, nr_pages, altmap, pgmap);
+ vmem_remove_mapping(start, size);
+ }
+ #endif /* CONFIG_MEMORY_HOTPLUG */
+diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
+index 606c74f274593e..80ff6c64f939ac 100644
+--- a/arch/x86/boot/compressed/Makefile
++++ b/arch/x86/boot/compressed/Makefile
+@@ -27,6 +27,7 @@ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
+ KBUILD_CFLAGS := -m$(BITS) -O2 $(CLANG_FLAGS)
+ KBUILD_CFLAGS += -std=gnu11
+ KBUILD_CFLAGS += -fno-strict-aliasing -fPIE
++KBUILD_CFLAGS += -fno-jump-tables
+ KBUILD_CFLAGS += -Wundef
+ KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
+ cflags-$(CONFIG_X86_32) := -march=i386
+diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
+index 2d48db66fca857..3db8992d1ae5f4 100644
+--- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
++++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
+@@ -2922,10 +2922,6 @@ static void rmdir_all_sub(void)
+ if (rdtgrp == &rdtgroup_default)
+ continue;
+
+- if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
+- rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)
+- rdtgroup_pseudo_lock_remove(rdtgrp);
+-
+ /*
+ * Give any CPUs back to the default group. We cannot copy
+ * cpu_online_mask because a CPU might have executed the
+@@ -2934,7 +2930,13 @@ static void rmdir_all_sub(void)
+ cpumask_or(&rdtgroup_default.cpu_mask,
+ &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
+
+- free_rmid(rdtgrp->closid, rdtgrp->mon.rmid);
++ if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
++ rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
++ rdtgroup_pseudo_lock_remove(rdtgrp);
++ } else {
++ /* Pseudo-locked group's RMID is freed during setup. */
++ free_rmid(rdtgrp->closid, rdtgrp->mon.rmid);
++ }
+
+ kernfs_remove(rdtgrp->kn);
+ list_del(&rdtgrp->rdtgroup_list);
+diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
+index ef9c8135311dc5..a7d38b8cbd7566 100644
+--- a/arch/x86/kvm/lapic.c
++++ b/arch/x86/kvm/lapic.c
+@@ -1923,7 +1923,7 @@ static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn)
+ if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
+ ktimer->expired_tscdeadline = ktimer->tscdeadline;
+
+- if (!from_timer_fn && apic->apicv_active) {
++ if (!from_timer_fn && apic->apicv_active && vcpu->wants_to_run) {
+ WARN_ON(kvm_get_running_vcpu() != vcpu);
+ kvm_apic_inject_pending_timer_irqs(apic);
+ return;
+diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
+index aab26f90c28551..9be2b10ff4f00c 100644
+--- a/arch/x86/kvm/mmu/mmu.c
++++ b/arch/x86/kvm/mmu/mmu.c
+@@ -4601,16 +4601,17 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
+
+ orig_pfn = fault->pfn;
+
+- r = RET_PF_RETRY;
+ write_lock(&vcpu->kvm->mmu_lock);
+
+- if (is_page_fault_stale(vcpu, fault))
+- goto out_unlock;
+-
+ r = make_mmu_pages_available(vcpu);
+ if (r)
+ goto out_unlock;
+
++ if (is_page_fault_stale(vcpu, fault)) {
++ r = RET_PF_RETRY;
++ goto out_unlock;
++ }
++
+ r = direct_map(vcpu, fault);
+
+ out_unlock:
+@@ -7229,7 +7230,9 @@ static struct shrinker *mmu_shrinker;
+ static void mmu_destroy_caches(void)
+ {
+ kmem_cache_destroy(pte_list_desc_cache);
++ pte_list_desc_cache = NULL;
+ kmem_cache_destroy(mmu_page_header_cache);
++ mmu_page_header_cache = NULL;
+ }
+
+ static void kvm_wake_nx_recovery_thread(struct kvm *kvm)
+diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
+index b08017683920f0..ce467bc3383c99 100644
+--- a/arch/x86/kvm/mmu/paging_tmpl.h
++++ b/arch/x86/kvm/mmu/paging_tmpl.h
+@@ -839,15 +839,17 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
+
+ orig_pfn = fault->pfn;
+
+- r = RET_PF_RETRY;
+ write_lock(&vcpu->kvm->mmu_lock);
+
+- if (is_page_fault_stale(vcpu, fault))
+- goto out_unlock;
+-
+ r = make_mmu_pages_available(vcpu);
+ if (r)
+ goto out_unlock;
++
++ if (is_page_fault_stale(vcpu, fault)) {
++ r = RET_PF_RETRY;
++ goto out_unlock;
++ }
++
+ r = FNAME(fetch)(vcpu, fault, &walker);
+
+ out_unlock:
+diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
+index 96edd76c1e39a6..44819d9470dcfa 100644
+--- a/arch/x86/kvm/svm/svm.c
++++ b/arch/x86/kvm/svm/svm.c
+@@ -626,7 +626,12 @@ static int svm_enable_virtualization_cpu(void)
+ return -EBUSY;
+
+ sd = per_cpu_ptr(&svm_data, me);
+- sd->asid_generation = 1;
++ /*
++ * Bump the current asid_generation value to ensure any vCPU that
++ * previously ran on this CPU sees a stale generation and is forced
++ * to acquire a new ASID, preventing a latent ASID collision.
++ */
++ sd->asid_generation++;
+ sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
+ sd->next_asid = sd->max_asid + 1;
+ sd->min_asid = max_sev_asid + 1;
+diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
+index 1a7a12af4a3a84..40078d5e6b2917 100644
+--- a/arch/x86/kvm/vmx/nested.c
++++ b/arch/x86/kvm/vmx/nested.c
+@@ -325,6 +325,7 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
+ static void free_nested(struct kvm_vcpu *vcpu)
+ {
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
++ struct vmcs *shadow_vmcs;
+
+ if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01))
+ vmx_switch_vmcs(vcpu, &vmx->vmcs01);
+@@ -342,9 +343,15 @@ static void free_nested(struct kvm_vcpu *vcpu)
+ vmx->nested.current_vmptr = INVALID_GPA;
+ if (enable_shadow_vmcs) {
+ vmx_disable_shadow_vmcs(vmx);
+- vmcs_clear(vmx->vmcs01.shadow_vmcs);
+- free_vmcs(vmx->vmcs01.shadow_vmcs);
++
++ /*
++ * Keep the pointer visible until after VMCLEAR, so migration
++ * can clear an active shadow VMCS on the old CPU.
++ */
++ shadow_vmcs = vmx->vmcs01.shadow_vmcs;
++ vmcs_clear(shadow_vmcs);
+ vmx->vmcs01.shadow_vmcs = NULL;
++ free_vmcs(shadow_vmcs);
+ }
+ kfree(vmx->nested.cached_vmcs12);
+ vmx->nested.cached_vmcs12 = NULL;
+diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
+index 63de8dbaa75ee8..8971bf1f4d8326 100644
+--- a/arch/x86/mm/init_64.c
++++ b/arch/x86/mm/init_64.c
+@@ -1293,12 +1293,13 @@ kernel_physical_mapping_remove(unsigned long start, unsigned long end)
+ remove_pagetable(start, end, true, NULL);
+ }
+
+-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
++void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
++ struct dev_pagemap *pgmap)
+ {
+ unsigned long start_pfn = start >> PAGE_SHIFT;
+ unsigned long nr_pages = size >> PAGE_SHIFT;
+
+- __remove_pages(start_pfn, nr_pages, altmap);
++ __remove_pages(start_pfn, nr_pages, altmap, pgmap);
+ kernel_physical_mapping_remove(start, start + size);
+ }
+ #endif /* CONFIG_MEMORY_HOTPLUG */
+diff --git a/block/genhd.c b/block/genhd.c
+index 22fea4eb424192..31e1539e6e6aa0 100644
+--- a/block/genhd.c
++++ b/block/genhd.c
+@@ -382,6 +382,28 @@ int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode)
+ return ret;
+ }
+
++static void add_disk_final(struct gendisk *disk)
++{
++ struct device *ddev = disk_to_dev(disk);
++
++ if (!(disk->flags & GENHD_FL_HIDDEN)) {
++ bdev_add(disk->part0, ddev->devt);
++ if (get_capacity(disk))
++ disk_scan_partitions(disk, BLK_OPEN_READ);
++
++ /*
++ * Announce the disk and partitions after all partitions are
++ * created. (for hidden disks uevents remain suppressed forever)
++ */
++ dev_set_uevent_suppress(ddev, 0);
++ disk_uevent(disk, KOBJ_ADD);
++ }
++
++ blk_apply_bdi_limits(disk->bdi, &disk->queue->limits);
++ disk_add_events(disk);
++ set_bit(GD_ADDED, &disk->state);
++}
++
+ /**
+ * device_add_disk - add disk information to kernel list
+ * @parent: parent device for the disk
+@@ -500,21 +522,6 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
+ &disk->bdi->dev->kobj, "bdi");
+ if (ret)
+ goto out_unregister_bdi;
+-
+- /* Make sure the first partition scan will be proceed */
+- if (get_capacity(disk) && disk_has_partscan(disk))
+- set_bit(GD_NEED_PART_SCAN, &disk->state);
+-
+- bdev_add(disk->part0, ddev->devt);
+- if (get_capacity(disk))
+- disk_scan_partitions(disk, BLK_OPEN_READ);
+-
+- /*
+- * Announce the disk and partitions after all partitions are
+- * created. (for hidden disks uevents remain suppressed forever)
+- */
+- dev_set_uevent_suppress(ddev, 0);
+- disk_uevent(disk, KOBJ_ADD);
+ } else {
+ /*
+ * Even if the block_device for a hidden gendisk is not
+@@ -523,10 +530,7 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
+ */
+ disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor);
+ }
+-
+- blk_apply_bdi_limits(disk->bdi, &disk->queue->limits);
+- disk_add_events(disk);
+- set_bit(GD_ADDED, &disk->state);
++ add_disk_final(disk);
+ return 0;
+
+ out_unregister_bdi:
+diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c
+index cd501195f34a1a..225fcc3377dddc 100644
+--- a/crypto/rsa-pkcs1pad.c
++++ b/crypto/rsa-pkcs1pad.c
+@@ -557,7 +557,7 @@ static int pkcs1pad_verify(struct akcipher_request *req)
+ const unsigned int digest_size = req->dst_len;
+ int err;
+
+- if (WARN_ON(req->dst) || WARN_ON(!digest_size) ||
++ if (WARN_ON(req->dst) || !digest_size ||
+ !ctx->key_size || sig_size != ctx->key_size)
+ return -EINVAL;
+
+diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c
+index 4af1b164d85a73..c9e1922aff5e84 100644
+--- a/drivers/accel/ivpu/ivpu_hw_btrs.c
++++ b/drivers/accel/ivpu/ivpu_hw_btrs.c
+@@ -852,7 +852,7 @@ static void diagnose_failure_mtl(struct ivpu_device *vdev)
+
+ static void diagnose_failure_lnl(struct ivpu_device *vdev)
+ {
+- u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
++ u32 reg = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
+
+ if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) {
+ ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
+diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
+index e216409f84ccaf..89cb03a5e8888b 100644
+--- a/drivers/ata/libata-core.c
++++ b/drivers/ata/libata-core.c
+@@ -2804,6 +2804,24 @@ static void ata_dev_config_cpr(struct ata_device *dev)
+ if (!nr_cpr)
+ goto out;
+
++ /*
++ * The device reports the number of CPR descriptors independently of the
++ * log size, and that count is also used to emit VPD page B9h into the
++ * fixed-size rbuf. Reject a count larger than what that buffer can hold
++ * (ATA_DEV_MAX_CPR) or larger than the log the device actually returned.
++ */
++ if (nr_cpr > ATA_DEV_MAX_CPR) {
++ ata_dev_warn(dev,
++ "Too many concurrent positioning ranges\n");
++ goto out;
++ }
++
++ if (buf_len < 64 + (size_t)nr_cpr * 32) {
++ ata_dev_warn(dev,
++ "Invalid number of concurrent positioning ranges\n");
++ goto out;
++ }
++
+ cpr_log = kzalloc(struct_size(cpr_log, cpr, nr_cpr), GFP_KERNEL);
+ if (!cpr_log)
+ goto out;
+diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
+index 4862d327c47aca..710d1d66afb12d 100644
+--- a/drivers/ata/libata-scsi.c
++++ b/drivers/ata/libata-scsi.c
+@@ -37,8 +37,6 @@
+ #include "libata.h"
+ #include "libata-transport.h"
+
+-#define ATA_SCSI_RBUF_SIZE 2048
+-
+ static DEFINE_SPINLOCK(ata_scsi_rbuf_lock);
+ static u8 ata_scsi_rbuf[ATA_SCSI_RBUF_SIZE];
+
+diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
+index 1a2d0f7115b5f8..1415190c18544e 100644
+--- a/drivers/ata/libata.h
++++ b/drivers/ata/libata.h
+@@ -145,6 +145,15 @@ static inline void ata_acpi_bind_dev(struct ata_device *dev) {}
+ #endif
+
+ /* libata-scsi.c */
++#define ATA_SCSI_RBUF_SIZE 2048
++
++/*
++ * Maximum number of concurrent positioning ranges (CPR) supported. The ACS
++ * specifications allow up to 255, but we limit this to the number of CPR
++ * descriptors that fit in the rbuf buffer used to emit VPD page B9h.
++ */
++#define ATA_DEV_MAX_CPR min(255, ((ATA_SCSI_RBUF_SIZE - 64) / 32))
++
+ extern struct ata_device *ata_scsi_find_dev(struct ata_port *ap,
+ const struct scsi_device *scsidev);
+ extern int ata_scsi_add_hosts(struct ata_host *host,
+diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c
+index 52f5168e4db542..66db8dd8ec2670 100644
+--- a/drivers/ata/sata_dwc_460ex.c
++++ b/drivers/ata/sata_dwc_460ex.c
+@@ -19,7 +19,6 @@
+ #include <linux/device.h>
+ #include <linux/dmaengine.h>
+ #include <linux/of.h>
+-#include <linux/of_irq.h>
+ #include <linux/platform_device.h>
+ #include <linux/phy/phy.h>
+ #include <linux/libata.h>
+@@ -226,7 +225,6 @@ static int sata_dwc_dma_init_old(struct platform_device *pdev,
+ struct sata_dwc_device *hsdev)
+ {
+ struct device *dev = &pdev->dev;
+- struct device_node *np = dev->of_node;
+
+ hsdev->dma = devm_kzalloc(dev, sizeof(*hsdev->dma), GFP_KERNEL);
+ if (!hsdev->dma)
+@@ -236,11 +234,9 @@ static int sata_dwc_dma_init_old(struct platform_device *pdev,
+ hsdev->dma->id = pdev->id;
+
+ /* Get SATA DMA interrupt number */
+- hsdev->dma->irq = irq_of_parse_and_map(np, 1);
+- if (!hsdev->dma->irq) {
+- dev_err(dev, "no SATA DMA irq\n");
+- return -ENODEV;
+- }
++ hsdev->dma->irq = platform_get_irq(pdev, 1);
++ if (hsdev->dma->irq < 0)
++ return hsdev->dma->irq;
+
+ /* Get physical SATA DMA register base address */
+ hsdev->dma->regs = devm_platform_ioremap_resource(pdev, 1);
+@@ -398,8 +394,7 @@ static void clear_serror(struct ata_port *ap)
+
+ static void clear_interrupt_bit(struct sata_dwc_device *hsdev, u32 bit)
+ {
+- sata_dwc_writel(&hsdev->sata_dwc_regs->intpr,
+- sata_dwc_readl(&hsdev->sata_dwc_regs->intpr));
++ sata_dwc_writel(&hsdev->sata_dwc_regs->intpr, bit);
+ }
+
+ static u32 qcmd_tag_to_mask(u8 tag)
+@@ -612,14 +607,9 @@ DRVSTILLBUSY:
+ status = ap->ops->sff_check_status(ap);
+ dev_dbg(ap->dev, "%s ATA status register=0x%x\n", __func__, status);
+
+- tag = 0;
+ while (tag_mask) {
+- while (!(tag_mask & 0x00000001)) {
+- tag++;
+- tag_mask <<= 1;
+- }
+-
+- tag_mask &= (~0x00000001);
++ tag = __ffs(tag_mask);
++ tag_mask &= ~(1U << tag);
+ qc = ata_qc_from_tag(ap, tag);
+ if (unlikely(!qc)) {
+ dev_err(ap->dev, "failed to get qc");
+@@ -1125,7 +1115,6 @@ static const struct ata_port_info sata_dwc_port_info[] = {
+ static int sata_dwc_probe(struct platform_device *ofdev)
+ {
+ struct device *dev = &ofdev->dev;
+- struct device_node *np = dev->of_node;
+ struct sata_dwc_device *hsdev;
+ u32 idr, versionr;
+ char *ver = (char *)&versionr;
+@@ -1168,18 +1157,13 @@ static int sata_dwc_probe(struct platform_device *ofdev)
+ /* Save dev for later use in dev_xxx() routines */
+ hsdev->dev = dev;
+
+- /* Enable SATA Interrupts */
+- sata_dwc_enable_interrupts(hsdev);
+-
+ /* Get SATA interrupt number */
+- irq = irq_of_parse_and_map(np, 0);
+- if (!irq) {
+- dev_err(dev, "no SATA DMA irq\n");
+- return -ENODEV;
+- }
++ irq = platform_get_irq(ofdev, 0);
++ if (irq < 0)
++ return irq;
+
+ #ifdef CONFIG_SATA_DWC_OLD_DMA
+- if (!of_property_present(np, "dmas")) {
++ if (!of_property_present(dev->of_node, "dmas")) {
+ err = sata_dwc_dma_init_old(ofdev, hsdev);
+ if (err)
+ return err;
+@@ -1203,6 +1187,8 @@ static int sata_dwc_probe(struct platform_device *ofdev)
+ if (err)
+ dev_err(dev, "failed to activate host");
+
++ /* Enable SATA Interrupts */
++ sata_dwc_enable_interrupts(hsdev);
+ return 0;
+
+ error_out:
+diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
+index b805614f6ae69e..46f9f854008679 100644
+--- a/drivers/block/rbd.c
++++ b/drivers/block/rbd.c
+@@ -1957,9 +1957,14 @@ static int rbd_object_map_update_finish(struct rbd_obj_request *obj_req,
+ bool has_current_state;
+ void *p;
+
+- if (osd_req->r_result)
++ if (osd_req->r_result < 0)
+ return osd_req->r_result;
+
++ /*
++ * Writes aren't allowed to return a data payload.
++ */
++ WARN_ON_ONCE(osd_req->r_result > 0);
++
+ /*
+ * Nothing to do for a snapshot object map.
+ */
+diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
+index 372427747cd649..083a599eec74db 100644
+--- a/drivers/bluetooth/btqca.c
++++ b/drivers/bluetooth/btqca.c
+@@ -413,7 +413,7 @@ static int qca_tlv_check_data(struct hci_dev *hdev,
+
+ idx = 0;
+ data = tlv->data;
+- while (idx < length - sizeof(struct tlv_type_nvm)) {
++ while (idx + sizeof(struct tlv_type_nvm) <= length) {
+ tlv_nvm = (struct tlv_type_nvm *)(data + idx);
+
+ tag_id = le16_to_cpu(tlv_nvm->tag_id);
+diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
+index 903d43380d038f..aadbda2fd5a20a 100644
+--- a/drivers/bluetooth/btusb.c
++++ b/drivers/bluetooth/btusb.c
+@@ -2707,7 +2707,9 @@ static int btusb_setup_realtek(struct hci_dev *hdev)
+
+ static int btusb_recv_event_realtek(struct hci_dev *hdev, struct sk_buff *skb)
+ {
+- if (skb->data[0] == HCI_VENDOR_PKT && skb->data[2] == RTK_SUB_EVENT_CODE_COREDUMP) {
++ if (skb->len >= HCI_EVENT_HDR_SIZE + 1 &&
++ skb->data[0] == HCI_VENDOR_PKT &&
++ skb->data[2] == RTK_SUB_EVENT_CODE_COREDUMP) {
+ struct rtk_dev_coredump_hdr hdr = {
+ .code = RTK_DEVCOREDUMP_CODE_MEMDUMP,
+ };
+diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
+index de9fd0e77d398a..524fc638ceb49f 100644
+--- a/drivers/bluetooth/hci_qca.c
++++ b/drivers/bluetooth/hci_qca.c
+@@ -1070,6 +1070,10 @@ static void qca_controller_memdump(struct work_struct *work)
+ if (!(qca_memdump->ram_dump_size)) {
+ bt_dev_err(hu->hdev, "Rx invalid memdump size");
+ kfree(qca_memdump);
++ qca->qca_memdump = NULL;
++ qca->memdump_state = QCA_MEMDUMP_COLLECTED;
++ clear_and_wake_up_bit(QCA_MEMDUMP_COLLECTION, &qca->flags);
++ clear_bit(QCA_IBS_DISABLED, &qca->flags);
+ kfree_skb(skb);
+ mutex_unlock(&qca->hci_memdump_lock);
+ return;
+diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c
+index 19d6f9a069bdf1..6a94024593e64f 100644
+--- a/drivers/cdrom/cdrom.c
++++ b/drivers/cdrom/cdrom.c
+@@ -3191,6 +3191,7 @@ static noinline int mmc_ioctl_cdrom_volume(struct cdrom_device_info *cdi,
+
+ /* set volume */
+ cgc->buffer = buffer + offset - 8;
++ cgc->buflen -= offset - 8;
+ memset(cgc->buffer, 0, 8);
+ return cdrom_mode_select(cdi, cgc);
+ }
+diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
+index 41ae4dac4eebad..8357cc5884b58a 100644
+--- a/drivers/char/ipmi/ipmi_msghandler.c
++++ b/drivers/char/ipmi/ipmi_msghandler.c
+@@ -2331,6 +2331,10 @@ static int i_ipmi_request(struct ipmi_user *user,
+ if (smi_msg == NULL) {
+ if (!supplied_recv)
+ ipmi_free_recv_msg(recv_msg);
++ else if (recv_msg->user) {
++ atomic_dec(&recv_msg->user->nr_msgs);
++ kref_put(&recv_msg->user->refcount, free_user);
++ }
+ return -ENOMEM;
+ }
+ }
+@@ -2373,6 +2377,10 @@ out_err:
+ ipmi_free_smi_msg(smi_msg);
+ if (!supplied_recv)
+ ipmi_free_recv_msg(recv_msg);
++ else if (recv_msg->user) {
++ atomic_dec(&recv_msg->user->nr_msgs);
++ kref_put(&recv_msg->user->refcount, free_user);
++ }
+ } else {
+ dev_dbg(intf->si_dev, "Send: %*ph\n",
+ smi_msg->data_size, smi_msg->data);
+diff --git a/drivers/comedi/drivers/comedi_parport.c b/drivers/comedi/drivers/comedi_parport.c
+index 098738a688fe64..db9f58792ab919 100644
+--- a/drivers/comedi/drivers/comedi_parport.c
++++ b/drivers/comedi/drivers/comedi_parport.c
+@@ -211,6 +211,13 @@ static irqreturn_t parport_interrupt(int irq, void *d)
+ unsigned int ctrl;
+ unsigned short val = 0;
+
++ /*
++ * Check device is fully attached. Device interrupts should have
++ * been disabled, but do this in case of bad hardware.
++ */
++ if (!dev->attached)
++ return IRQ_NONE;
++
+ ctrl = inb(dev->iobase + PARPORT_CTRL_REG);
+ if (!(ctrl & PARPORT_CTRL_IRQ_ENA))
+ return IRQ_NONE;
+@@ -231,6 +238,9 @@ static int parport_attach(struct comedi_device *dev,
+ if (ret)
+ return ret;
+
++ outb(0, dev->iobase + PARPORT_DATA_REG);
++ outb(0, dev->iobase + PARPORT_CTRL_REG);
++
+ if (it->options[1]) {
+ ret = request_irq(it->options[1], parport_interrupt, 0,
+ dev->board_name, dev);
+@@ -286,9 +296,6 @@ static int parport_attach(struct comedi_device *dev,
+ s->cancel = parport_intr_cancel;
+ }
+
+- outb(0, dev->iobase + PARPORT_DATA_REG);
+- outb(0, dev->iobase + PARPORT_CTRL_REG);
+-
+ return 0;
+ }
+
+diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
+index 17df96564ad6f2..2c86539eb4bc38 100644
+--- a/drivers/cpufreq/cpufreq.c
++++ b/drivers/cpufreq/cpufreq.c
+@@ -2601,6 +2601,9 @@ static void cpufreq_update_pressure(struct cpufreq_policy *policy)
+
+ cpu = cpumask_first(policy->related_cpus);
+ max_freq = arch_scale_freq_ref(cpu);
++ if (!max_freq)
++ max_freq = policy->cpuinfo.max_freq;
++
+ capped_freq = policy->max;
+
+ /*
+diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c
+index 3c3b9f18d8cf50..e5ff4fe66a4b88 100644
+--- a/drivers/crypto/tegra/tegra-se-main.c
++++ b/drivers/crypto/tegra/tegra-se-main.c
+@@ -52,7 +52,7 @@ tegra_se_cmdbuf_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_dire
+ return ERR_PTR(-ENOMEM);
+
+ kref_init(&map->ref);
+- map->bo = host1x_bo_get(bo);
++ map->bo = bo;
+ map->direction = direction;
+ map->dev = dev;
+
+@@ -93,7 +93,6 @@ static void tegra_se_cmdbuf_unpin(struct host1x_bo_mapping *map)
+ dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);
+ sg_free_table(map->sgt);
+ kfree(map->sgt);
+- host1x_bo_put(map->bo);
+
+ kfree(map);
+ }
+diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c
+index 5c92834f3fc86d..b81fd91e30e777 100644
+--- a/drivers/dma-buf/udmabuf.c
++++ b/drivers/dma-buf/udmabuf.c
+@@ -207,21 +207,22 @@ static int begin_cpu_udmabuf(struct dma_buf *buf,
+ {
+ struct udmabuf *ubuf = buf->priv;
+ struct device *dev = ubuf->device->this_device;
+- int ret = 0;
+
+ if (!ubuf->sg) {
+ ubuf->sg = get_sg_table(dev, buf, direction);
+ if (IS_ERR(ubuf->sg)) {
++ int ret;
++
+ ret = PTR_ERR(ubuf->sg);
+ ubuf->sg = NULL;
++ return ret;
+ } else {
+ ubuf->sg_dir = direction;
+ }
+- } else {
+- dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction);
+ }
+
+- return ret;
++ dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction);
++ return 0;
+ }
+
+ static int end_cpu_udmabuf(struct dma_buf *buf,
+diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
+index 1c6043751dc925..b3f2653862a8a5 100644
+--- a/drivers/dma/dw-edma/dw-edma-pcie.c
++++ b/drivers/dma/dw-edma/dw-edma-pcie.c
+@@ -160,12 +160,19 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
+ const struct pci_device_id *pid)
+ {
+ struct dw_edma_pcie_data *pdata = (void *)pid->driver_data;
+- struct dw_edma_pcie_data vsec_data;
+ struct device *dev = &pdev->dev;
+ struct dw_edma_chip *chip;
+ int err, nr_irqs;
+ int i, mask;
+
++ if (!pdata)
++ return -ENODEV;
++
++ struct dw_edma_pcie_data *vsec_data __free(kfree) =
++ kmalloc(sizeof(*vsec_data), GFP_KERNEL);
++ if (!vsec_data)
++ return -ENOMEM;
++
+ /* Enable PCI device */
+ err = pcim_enable_device(pdev);
+ if (err) {
+@@ -173,23 +180,23 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
+ return err;
+ }
+
+- memcpy(&vsec_data, pdata, sizeof(struct dw_edma_pcie_data));
++ memcpy(vsec_data, pdata, sizeof(struct dw_edma_pcie_data));
+
+ /*
+ * Tries to find if exists a PCIe Vendor-Specific Extended Capability
+ * for the DMA, if one exists, then reconfigures it.
+ */
+- dw_edma_pcie_get_vsec_dma_data(pdev, &vsec_data);
++ dw_edma_pcie_get_vsec_dma_data(pdev, vsec_data);
+
+ /* Mapping PCI BAR regions */
+- mask = BIT(vsec_data.rg.bar);
+- for (i = 0; i < vsec_data.wr_ch_cnt; i++) {
+- mask |= BIT(vsec_data.ll_wr[i].bar);
+- mask |= BIT(vsec_data.dt_wr[i].bar);
++ mask = BIT(vsec_data->rg.bar);
++ for (i = 0; i < vsec_data->wr_ch_cnt; i++) {
++ mask |= BIT(vsec_data->ll_wr[i].bar);
++ mask |= BIT(vsec_data->dt_wr[i].bar);
+ }
+- for (i = 0; i < vsec_data.rd_ch_cnt; i++) {
+- mask |= BIT(vsec_data.ll_rd[i].bar);
+- mask |= BIT(vsec_data.dt_rd[i].bar);
++ for (i = 0; i < vsec_data->rd_ch_cnt; i++) {
++ mask |= BIT(vsec_data->ll_rd[i].bar);
++ mask |= BIT(vsec_data->dt_rd[i].bar);
+ }
+ err = pcim_iomap_regions(pdev, mask, pci_name(pdev));
+ if (err) {
+@@ -212,7 +219,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
+ return -ENOMEM;
+
+ /* IRQs allocation */
+- nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data.irqs,
++ nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data->irqs,
+ PCI_IRQ_MSI | PCI_IRQ_MSIX);
+ if (nr_irqs < 1) {
+ pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n",
+@@ -223,22 +230,22 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
+ /* Data structure initialization */
+ chip->dev = dev;
+
+- chip->mf = vsec_data.mf;
++ chip->mf = vsec_data->mf;
+ chip->nr_irqs = nr_irqs;
+ chip->ops = &dw_edma_pcie_plat_ops;
+
+- chip->ll_wr_cnt = vsec_data.wr_ch_cnt;
+- chip->ll_rd_cnt = vsec_data.rd_ch_cnt;
++ chip->ll_wr_cnt = vsec_data->wr_ch_cnt;
++ chip->ll_rd_cnt = vsec_data->rd_ch_cnt;
+
+- chip->reg_base = pcim_iomap_table(pdev)[vsec_data.rg.bar];
++ chip->reg_base = pcim_iomap_table(pdev)[vsec_data->rg.bar];
+ if (!chip->reg_base)
+ return -ENOMEM;
+
+ for (i = 0; i < chip->ll_wr_cnt; i++) {
+ struct dw_edma_region *ll_region = &chip->ll_region_wr[i];
+ struct dw_edma_region *dt_region = &chip->dt_region_wr[i];
+- struct dw_edma_block *ll_block = &vsec_data.ll_wr[i];
+- struct dw_edma_block *dt_block = &vsec_data.dt_wr[i];
++ struct dw_edma_block *ll_block = &vsec_data->ll_wr[i];
++ struct dw_edma_block *dt_block = &vsec_data->dt_wr[i];
+
+ ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar];
+ if (!ll_region->vaddr.io)
+@@ -262,8 +269,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
+ for (i = 0; i < chip->ll_rd_cnt; i++) {
+ struct dw_edma_region *ll_region = &chip->ll_region_rd[i];
+ struct dw_edma_region *dt_region = &chip->dt_region_rd[i];
+- struct dw_edma_block *ll_block = &vsec_data.ll_rd[i];
+- struct dw_edma_block *dt_block = &vsec_data.dt_rd[i];
++ struct dw_edma_block *ll_block = &vsec_data->ll_rd[i];
++ struct dw_edma_block *dt_block = &vsec_data->dt_rd[i];
+
+ ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar];
+ if (!ll_region->vaddr.io)
+@@ -295,31 +302,31 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
+ pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", chip->mf);
+
+ pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n",
+- vsec_data.rg.bar, vsec_data.rg.off, vsec_data.rg.sz,
++ vsec_data->rg.bar, vsec_data->rg.off, vsec_data->rg.sz,
+ chip->reg_base);
+
+
+ for (i = 0; i < chip->ll_wr_cnt; i++) {
+ pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
+- i, vsec_data.ll_wr[i].bar,
+- vsec_data.ll_wr[i].off, chip->ll_region_wr[i].sz,
++ i, vsec_data->ll_wr[i].bar,
++ vsec_data->ll_wr[i].off, chip->ll_region_wr[i].sz,
+ chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr);
+
+ pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
+- i, vsec_data.dt_wr[i].bar,
+- vsec_data.dt_wr[i].off, chip->dt_region_wr[i].sz,
++ i, vsec_data->dt_wr[i].bar,
++ vsec_data->dt_wr[i].off, chip->dt_region_wr[i].sz,
+ chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr);
+ }
+
+ for (i = 0; i < chip->ll_rd_cnt; i++) {
+ pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
+- i, vsec_data.ll_rd[i].bar,
+- vsec_data.ll_rd[i].off, chip->ll_region_rd[i].sz,
++ i, vsec_data->ll_rd[i].bar,
++ vsec_data->ll_rd[i].off, chip->ll_region_rd[i].sz,
+ chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr);
+
+ pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
+- i, vsec_data.dt_rd[i].bar,
+- vsec_data.dt_rd[i].off, chip->dt_region_rd[i].sz,
++ i, vsec_data->dt_rd[i].bar,
++ vsec_data->dt_rd[i].off, chip->dt_region_rd[i].sz,
+ chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr);
+ }
+
+diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c
+index 06952519f509b8..fece2cfec1ccb4 100644
+--- a/drivers/dma/sh/rz-dmac.c
++++ b/drivers/dma/sh/rz-dmac.c
+@@ -780,27 +780,6 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac,
+ channel->index = index;
+ channel->mid_rid = -EINVAL;
+
+- /* Request the channel interrupt. */
+- scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);
+- channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
+- if (channel->irq < 0)
+- return channel->irq;
+-
+- irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
+- dev_name(dmac->dev), index);
+- if (!irqname)
+- return -ENOMEM;
+-
+- ret = devm_request_threaded_irq(dmac->dev, channel->irq,
+- rz_dmac_irq_handler,
+- rz_dmac_irq_handler_thread, 0,
+- irqname, channel);
+- if (ret) {
+- dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
+- channel->irq, ret);
+- return ret;
+- }
+-
+ /* Set io base address for each channel */
+ if (index < 8) {
+ channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
+@@ -813,9 +792,9 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac,
+ }
+
+ /* Allocate descriptors */
+- lmdesc = dma_alloc_coherent(&pdev->dev,
+- sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
+- &channel->lmdesc.base_dma, GFP_KERNEL);
++ lmdesc = dmam_alloc_coherent(&pdev->dev,
++ sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
++ &channel->lmdesc.base_dma, GFP_KERNEL);
+ if (!lmdesc) {
+ dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
+ return -ENOMEM;
+@@ -831,7 +810,26 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac,
+ INIT_LIST_HEAD(&channel->ld_free);
+ INIT_LIST_HEAD(&channel->ld_active);
+
+- return 0;
++ /* Request the channel interrupt. */
++ scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);
++ channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
++ if (channel->irq < 0)
++ return channel->irq;
++
++ irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
++ dev_name(dmac->dev), index);
++ if (!irqname)
++ return -ENOMEM;
++
++ ret = devm_request_threaded_irq(dmac->dev, channel->irq,
++ rz_dmac_irq_handler,
++ rz_dmac_irq_handler_thread, 0,
++ irqname, channel);
++ if (ret)
++ dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
++ channel->irq, ret);
++
++ return ret;
+ }
+
+ static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
+@@ -858,7 +856,6 @@ static int rz_dmac_probe(struct platform_device *pdev)
+ const char *irqname = "error";
+ struct dma_device *engine;
+ struct rz_dmac *dmac;
+- int channel_num;
+ int ret;
+ int irq;
+ u8 i;
+@@ -888,19 +885,6 @@ static int rz_dmac_probe(struct platform_device *pdev)
+ if (IS_ERR(dmac->ext_base))
+ return PTR_ERR(dmac->ext_base);
+
+- /* Register interrupt handler for error */
+- irq = platform_get_irq_byname(pdev, irqname);
+- if (irq < 0)
+- return irq;
+-
+- ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
+- irqname, NULL);
+- if (ret) {
+- dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
+- irq, ret);
+- return ret;
+- }
+-
+ /* Initialize the channels. */
+ INIT_LIST_HEAD(&dmac->engine.channels);
+
+@@ -926,6 +910,21 @@ static int rz_dmac_probe(struct platform_device *pdev)
+ goto err;
+ }
+
++ /* Register interrupt handler for error */
++ irq = platform_get_irq_byname(pdev, irqname);
++ if (irq < 0) {
++ ret = irq;
++ goto err;
++ }
++
++ ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
++ irqname, NULL);
++ if (ret) {
++ dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
++ irq, ret);
++ goto err;
++ }
++
+ /* Register the DMAC as a DMA provider for DT. */
+ ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
+ NULL);
+@@ -964,16 +963,6 @@ static int rz_dmac_probe(struct platform_device *pdev)
+ dma_register_err:
+ of_dma_controller_free(pdev->dev.of_node);
+ err:
+- channel_num = i ? i - 1 : 0;
+- for (i = 0; i < channel_num; i++) {
+- struct rz_dmac_chan *channel = &dmac->channels[i];
+-
+- dma_free_coherent(&pdev->dev,
+- sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
+- channel->lmdesc.base,
+- channel->lmdesc.base_dma);
+- }
+-
+ reset_control_assert(dmac->rstc);
+ err_pm_runtime_put:
+ pm_runtime_put(&pdev->dev);
+@@ -986,18 +975,9 @@ err_pm_disable:
+ static void rz_dmac_remove(struct platform_device *pdev)
+ {
+ struct rz_dmac *dmac = platform_get_drvdata(pdev);
+- unsigned int i;
+
+ dma_async_device_unregister(&dmac->engine);
+ of_dma_controller_free(pdev->dev.of_node);
+- for (i = 0; i < dmac->n_channels; i++) {
+- struct rz_dmac_chan *channel = &dmac->channels[i];
+-
+- dma_free_coherent(&pdev->dev,
+- sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
+- channel->lmdesc.base,
+- channel->lmdesc.base_dma);
+- }
+ reset_control_assert(dmac->rstc);
+ pm_runtime_put(&pdev->dev);
+ pm_runtime_disable(&pdev->dev);
+diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
+index e3fb08e1578d85..d6281bc46988a8 100644
+--- a/drivers/dpll/dpll_netlink.c
++++ b/drivers/dpll/dpll_netlink.c
+@@ -187,6 +187,27 @@ dpll_msg_add_temp(struct sk_buff *msg, struct dpll_device *dpll,
+ return 0;
+ }
+
++static int
++dpll_msg_add_clock_quality_level(struct sk_buff *msg, struct dpll_device *dpll,
++ struct netlink_ext_ack *extack)
++{
++ DECLARE_BITMAP(qls, DPLL_CLOCK_QUALITY_LEVEL_MAX + 1) = { 0 };
++ const struct dpll_device_ops *ops = dpll_device_ops(dpll);
++ enum dpll_clock_quality_level ql;
++ int ret;
++
++ if (!ops->clock_quality_level_get)
++ return 0;
++ ret = ops->clock_quality_level_get(dpll, dpll_priv(dpll), qls, extack);
++ if (ret)
++ return ret;
++ for_each_set_bit(ql, qls, DPLL_CLOCK_QUALITY_LEVEL_MAX + 1)
++ if (nla_put_u32(msg, DPLL_A_CLOCK_QUALITY_LEVEL, ql))
++ return -EMSGSIZE;
++
++ return 0;
++}
++
+ static int
+ dpll_msg_add_pin_prio(struct sk_buff *msg, struct dpll_pin *pin,
+ struct dpll_pin_ref *ref,
+@@ -623,6 +644,9 @@ dpll_device_get_one(struct dpll_device *dpll, struct sk_buff *msg,
+ if (ret)
+ return ret;
+ ret = dpll_msg_add_lock_status(msg, dpll, extack);
++ if (ret)
++ return ret;
++ ret = dpll_msg_add_clock_quality_level(msg, dpll, extack);
+ if (ret)
+ return ret;
+ ret = dpll_msg_add_mode(msg, dpll, extack);
+diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c
+index 423ead5fa9c135..a601a0130414ff 100644
+--- a/drivers/firewire/net.c
++++ b/drivers/firewire/net.c
+@@ -298,31 +298,34 @@ static struct fwnet_fragment_info *fwnet_frag_new(
+ if (fi->offset + fi->len == offset) {
+ /* The new fragment can be tacked on to the end */
+ /* Did the new fragment plug a hole? */
+- fi2 = list_entry(fi->fi_link.next,
+- struct fwnet_fragment_info, fi_link);
+- if (fi->offset + fi->len == fi2->offset) {
+- /* glue fragments together */
+- fi->len += len + fi2->len;
+- list_del(&fi2->fi_link);
+- kfree(fi2);
+- } else {
+- fi->len += len;
++ if (!list_is_last(&fi->fi_link, &pd->fi_list)) {
++ fi2 = list_next_entry(fi, fi_link);
++ if (offset + len == fi2->offset) {
++ /* glue fragments together */
++ fi->len += len + fi2->len;
++ list_del(&fi2->fi_link);
++ kfree(fi2);
++
++ return fi;
++ }
+ }
++ fi->len += len;
+
+ return fi;
+ }
+ if (offset + len == fi->offset) {
+ /* The new fragment can be tacked on to the beginning */
+ /* Did the new fragment plug a hole? */
+- fi2 = list_entry(fi->fi_link.prev,
+- struct fwnet_fragment_info, fi_link);
+- if (fi2->offset + fi2->len == fi->offset) {
+- /* glue fragments together */
+- fi2->len += fi->len + len;
+- list_del(&fi->fi_link);
+- kfree(fi);
+-
+- return fi2;
++ if (!list_is_first(&fi->fi_link, &pd->fi_list)) {
++ fi2 = list_prev_entry(fi, fi_link);
++ if (fi2->offset + fi2->len == offset) {
++ /* glue fragments together */
++ fi2->len += fi->len + len;
++ list_del(&fi->fi_link);
++ kfree(fi);
++
++ return fi2;
++ }
+ }
+ fi->offset = offset;
+ fi->len += len;
+diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
+index 3b221ba990aa1e..62222c9cfc3e6d 100644
+--- a/drivers/firmware/arm_ffa/driver.c
++++ b/drivers/firmware/arm_ffa/driver.c
+@@ -32,6 +32,7 @@
+ #include <linux/interrupt.h>
+ #include <linux/io.h>
+ #include <linux/kernel.h>
++#include <linux/minmax.h>
+ #include <linux/module.h>
+ #include <linux/mm.h>
+ #include <linux/mutex.h>
+@@ -55,7 +56,9 @@
+ (FIELD_PREP(SENDER_ID_MASK, (s)) | FIELD_PREP(RECEIVER_ID_MASK, (r)))
+
+ #define RXTX_MAP_MIN_BUFSZ_MASK GENMASK(1, 0)
+-#define RXTX_MAP_MIN_BUFSZ(x) ((x) & RXTX_MAP_MIN_BUFSZ_MASK)
++#define RXTX_MAP_MAX_BUFSZ_MASK GENMASK(31, 16)
++#define RXTX_MAP_MIN_BUFSZ(x) (FIELD_GET(RXTX_MAP_MIN_BUFSZ_MASK, (x)))
++#define RXTX_MAP_MAX_BUFSZ(x) (FIELD_GET(RXTX_MAP_MAX_BUFSZ_MASK, (x)))
+
+ #define FFA_MAX_NOTIFICATIONS 64
+
+@@ -1026,7 +1029,7 @@ static int ffa_partition_info_get(const char *uuid_str,
+ uuid_t uuid;
+ struct ffa_partition_info *pbuf;
+
+- if (uuid_parse(uuid_str, &uuid)) {
++ if (!uuid_str || uuid_parse(uuid_str, &uuid)) {
+ pr_err("invalid uuid (%s)\n", uuid_str);
+ return -ENODEV;
+ }
+@@ -1855,7 +1858,7 @@ static int __init ffa_init(void)
+ {
+ int ret;
+ u32 buf_sz;
+- size_t rxtx_bufsz = SZ_4K;
++ size_t rxtx_min_bufsz = SZ_4K, rxtx_max_bufsz = 0, rxtx_bufsz;
+
+ ret = ffa_transport_init(&invoke_ffa_fn);
+ if (ret)
+@@ -1878,15 +1881,18 @@ static int __init ffa_init(void)
+ ret = ffa_features(FFA_FN_NATIVE(RXTX_MAP), 0, &buf_sz, NULL);
+ if (!ret) {
+ if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 1)
+- rxtx_bufsz = SZ_64K;
++ rxtx_min_bufsz = SZ_64K;
+ else if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 2)
+- rxtx_bufsz = SZ_16K;
++ rxtx_min_bufsz = SZ_16K;
+ else
+- rxtx_bufsz = SZ_4K;
++ rxtx_min_bufsz = SZ_4K;
++
++ rxtx_max_bufsz = RXTX_MAP_MAX_BUFSZ(buf_sz) * SZ_4K;
++ if (rxtx_max_bufsz != 0 && rxtx_max_bufsz < rxtx_min_bufsz)
++ rxtx_max_bufsz = rxtx_min_bufsz;
+ }
+
+- rxtx_bufsz = PAGE_ALIGN(rxtx_bufsz);
+- drv_info->rxtx_bufsz = rxtx_bufsz;
++ rxtx_bufsz = min_not_zero(PAGE_ALIGN(rxtx_min_bufsz), rxtx_max_bufsz);
+ drv_info->rx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
+ if (!drv_info->rx_buffer) {
+ ret = -ENOMEM;
+@@ -1902,10 +1908,17 @@ static int __init ffa_init(void)
+ ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
+ virt_to_phys(drv_info->rx_buffer),
+ rxtx_bufsz / FFA_PAGE_SIZE);
++ if (ret == -EINVAL && !rxtx_max_bufsz && rxtx_min_bufsz < rxtx_bufsz) {
++ rxtx_bufsz = rxtx_min_bufsz;
++ ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
++ virt_to_phys(drv_info->rx_buffer),
++ rxtx_bufsz / FFA_PAGE_SIZE);
++ }
+ if (ret) {
+ pr_err("failed to register FFA RxTx buffers\n");
+ goto free_pages;
+ }
++ drv_info->rxtx_bufsz = rxtx_bufsz;
+
+ mutex_init(&drv_info->rx_lock);
+ mutex_init(&drv_info->tx_lock);
+diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c
+index e160ecb22948fc..f042fb8a050265 100644
+--- a/drivers/firmware/arm_scmi/notify.c
++++ b/drivers/firmware/arm_scmi/notify.c
+@@ -596,9 +596,9 @@ int scmi_notify(const struct scmi_handle *handle, u8 proto_id, u8 evt_id,
+ return -EINVAL;
+ }
+ if (kfifo_avail(&r_evt->proto->equeue.kfifo) < sizeof(eh) + len) {
+- dev_warn(handle->dev,
+- "queue full, dropping proto_id:%d evt_id:%d ts:%lld\n",
+- proto_id, evt_id, ktime_to_ns(ts));
++ dev_warn_ratelimited(handle->dev,
++ "queue full, dropping proto_id:%d evt_id:%d ts:%lld\n",
++ proto_id, evt_id, ktime_to_ns(ts));
+ return -ENOMEM;
+ }
+
+diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c
+index 3388b602b70b00..2d02f3f0b9338e 100644
+--- a/drivers/firmware/stratix10-svc.c
++++ b/drivers/firmware/stratix10-svc.c
+@@ -1083,14 +1083,16 @@ void *stratix10_svc_allocate_memory(struct stratix10_svc_chan *chan,
+ struct gen_pool *genpool = chan->ctrl->genpool;
+ size_t s = roundup(size, 1 << genpool->min_alloc_order);
+
+- pmem = devm_kzalloc(chan->ctrl->dev, sizeof(*pmem), GFP_KERNEL);
++ pmem = kzalloc_obj(*pmem);
+ if (!pmem)
+ return ERR_PTR(-ENOMEM);
+
+ guard(mutex)(&svc_mem_lock);
+ va = gen_pool_alloc(genpool, s);
+- if (!va)
++ if (!va) {
++ kfree(pmem);
+ return ERR_PTR(-ENOMEM);
++ }
+
+ memset((void *)va, 0, s);
+ pa = gen_pool_virt_to_phys(genpool, va);
+@@ -1116,6 +1118,7 @@ EXPORT_SYMBOL_GPL(stratix10_svc_allocate_memory);
+ void stratix10_svc_free_memory(struct stratix10_svc_chan *chan, void *kaddr)
+ {
+ struct stratix10_svc_data_mem *pmem;
++
+ guard(mutex)(&svc_mem_lock);
+
+ list_for_each_entry(pmem, &svc_data_mem, node)
+@@ -1124,10 +1127,9 @@ void stratix10_svc_free_memory(struct stratix10_svc_chan *chan, void *kaddr)
+ (unsigned long)kaddr, pmem->size);
+ pmem->vaddr = NULL;
+ list_del(&pmem->node);
++ kfree(pmem);
+ return;
+ }
+-
+- list_del(&svc_data_mem);
+ }
+ EXPORT_SYMBOL_GPL(stratix10_svc_free_memory);
+
+diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
+index 5578158f13750c..7d4c7c248d9c41 100644
+--- a/drivers/firmware/xilinx/zynqmp.c
++++ b/drivers/firmware/xilinx/zynqmp.c
+@@ -386,6 +386,7 @@ int zynqmp_pm_invoke_fn(u32 pm_api_id, u32 *ret_payload, u32 num_args, ...)
+ return do_fw_call(ret_payload, 8, smc_arg[0], smc_arg[1], smc_arg[2], smc_arg[3],
+ smc_arg[4], smc_arg[5], smc_arg[6], smc_arg[7]);
+ }
++EXPORT_SYMBOL_GPL(zynqmp_pm_invoke_fn);
+
+ static u32 pm_api_version;
+ static u32 pm_tz_version;
+diff --git a/drivers/fpga/dfl-afu-main.c b/drivers/fpga/dfl-afu-main.c
+index 6b97c073849ea9..89e922d9f6d583 100644
+--- a/drivers/fpga/dfl-afu-main.c
++++ b/drivers/fpga/dfl-afu-main.c
+@@ -720,6 +720,9 @@ afu_ioctl_dma_map(struct dfl_feature_platform_data *pdata, void __user *arg)
+ if (map.argsz < minsz || map.flags)
+ return -EINVAL;
+
++ if (map.length >> PAGE_SHIFT > (u64)INT_MAX)
++ return -EINVAL;
++
+ ret = afu_dma_map_region(pdata, map.user_addr, map.length, &map.iova);
+ if (ret)
+ return ret;
+diff --git a/drivers/gpio/gpio-mt7621.c b/drivers/gpio/gpio-mt7621.c
+index 93facbebb80efa..212a5f57cfd2bd 100644
+--- a/drivers/gpio/gpio-mt7621.c
++++ b/drivers/gpio/gpio-mt7621.c
+@@ -156,6 +156,8 @@ mediatek_gpio_irq_type(struct irq_data *d, unsigned int type)
+ int pin = d->hwirq;
+ u32 mask = BIT(pin);
+
++ guard(spinlock_irqsave)(&rg->lock);
++
+ if (type == IRQ_TYPE_PROBE) {
+ if ((rg->rising | rg->falling |
+ rg->hlevel | rg->llevel) & mask)
+diff --git a/drivers/gpu/Kconfig b/drivers/gpu/Kconfig
+new file mode 100644
+index 00000000000000..ebb2ad4b7ea0bb
+--- /dev/null
++++ b/drivers/gpu/Kconfig
+@@ -0,0 +1,13 @@
++# SPDX-License-Identifier: GPL-2.0
++
++config GPU_BUDDY
++ bool
++ help
++ A page based buddy allocator for GPU memory.
++
++config GPU_BUDDY_KUNIT_TEST
++ tristate "KUnit tests for GPU buddy allocator" if !KUNIT_ALL_TESTS
++ depends on GPU_BUDDY && KUNIT
++ default KUNIT_ALL_TESTS
++ help
++ KUnit tests for the GPU buddy allocator.
+diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile
+index 8997f0096545e2..f8810d75601939 100644
+--- a/drivers/gpu/Makefile
++++ b/drivers/gpu/Makefile
+@@ -2,6 +2,8 @@
+ # drm/tegra depends on host1x, so if both drivers are built-in care must be
+ # taken to initialize them in the correct order. Link order is the only way
+ # to ensure this currently.
+-obj-y += host1x/ drm/ vga/
++# Similarly, buddy must come first since it is used by other drivers.
++obj-$(CONFIG_GPU_BUDDY) += buddy.o
++obj-y += host1x/ drm/ vga/ tests/
+ obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/
+ obj-$(CONFIG_TRACE_GPU_MEM) += trace/
+diff --git a/drivers/gpu/buddy.c b/drivers/gpu/buddy.c
+new file mode 100644
+index 00000000000000..bf5920bd225803
+--- /dev/null
++++ b/drivers/gpu/buddy.c
+@@ -0,0 +1,1347 @@
++// SPDX-License-Identifier: MIT
++/*
++ * Copyright © 2021 Intel Corporation
++ */
++
++#include <kunit/test-bug.h>
++
++#include <linux/export.h>
++#include <linux/kmemleak.h>
++#include <linux/module.h>
++#include <linux/sizes.h>
++
++#include <linux/gpu_buddy.h>
++
++static struct kmem_cache *slab_blocks;
++
++static struct gpu_buddy_block *gpu_block_alloc(struct gpu_buddy *mm,
++ struct gpu_buddy_block *parent,
++ unsigned int order,
++ u64 offset)
++{
++ struct gpu_buddy_block *block;
++
++ BUG_ON(order > GPU_BUDDY_MAX_ORDER);
++
++ block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL);
++ if (!block)
++ return NULL;
++
++ block->header = offset;
++ block->header |= order;
++ block->parent = parent;
++
++ RB_CLEAR_NODE(&block->rb);
++
++ BUG_ON(block->header & GPU_BUDDY_HEADER_UNUSED);
++ return block;
++}
++
++static void gpu_block_free(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ kmem_cache_free(slab_blocks, block);
++}
++
++static enum gpu_buddy_free_tree
++get_block_tree(struct gpu_buddy_block *block)
++{
++ return gpu_buddy_block_is_clear(block) ?
++ GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
++}
++
++static struct gpu_buddy_block *
++rbtree_get_free_block(const struct rb_node *node)
++{
++ return node ? rb_entry(node, struct gpu_buddy_block, rb) : NULL;
++}
++
++static struct gpu_buddy_block *
++rbtree_last_free_block(struct rb_root *root)
++{
++ return rbtree_get_free_block(rb_last(root));
++}
++
++static bool rbtree_is_empty(struct rb_root *root)
++{
++ return RB_EMPTY_ROOT(root);
++}
++
++static bool gpu_buddy_block_offset_less(const struct gpu_buddy_block *block,
++ const struct gpu_buddy_block *node)
++{
++ return gpu_buddy_block_offset(block) < gpu_buddy_block_offset(node);
++}
++
++static bool rbtree_block_offset_less(struct rb_node *block,
++ const struct rb_node *node)
++{
++ return gpu_buddy_block_offset_less(rbtree_get_free_block(block),
++ rbtree_get_free_block(node));
++}
++
++static void rbtree_insert(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block,
++ enum gpu_buddy_free_tree tree)
++{
++ rb_add(&block->rb,
++ &mm->free_trees[tree][gpu_buddy_block_order(block)],
++ rbtree_block_offset_less);
++}
++
++static void rbtree_remove(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ unsigned int order = gpu_buddy_block_order(block);
++ enum gpu_buddy_free_tree tree;
++ struct rb_root *root;
++
++ tree = get_block_tree(block);
++ root = &mm->free_trees[tree][order];
++
++ rb_erase(&block->rb, root);
++ RB_CLEAR_NODE(&block->rb);
++}
++
++static void clear_reset(struct gpu_buddy_block *block)
++{
++ block->header &= ~GPU_BUDDY_HEADER_CLEAR;
++}
++
++static void mark_cleared(struct gpu_buddy_block *block)
++{
++ block->header |= GPU_BUDDY_HEADER_CLEAR;
++}
++
++static void mark_allocated(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ block->header &= ~GPU_BUDDY_HEADER_STATE;
++ block->header |= GPU_BUDDY_ALLOCATED;
++
++ rbtree_remove(mm, block);
++}
++
++static void mark_free(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ enum gpu_buddy_free_tree tree;
++
++ block->header &= ~GPU_BUDDY_HEADER_STATE;
++ block->header |= GPU_BUDDY_FREE;
++
++ tree = get_block_tree(block);
++ rbtree_insert(mm, block, tree);
++}
++
++static void mark_split(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ block->header &= ~GPU_BUDDY_HEADER_STATE;
++ block->header |= GPU_BUDDY_SPLIT;
++
++ rbtree_remove(mm, block);
++}
++
++static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
++{
++ return s1 <= e2 && e1 >= s2;
++}
++
++static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
++{
++ return s1 <= s2 && e1 >= e2;
++}
++
++static struct gpu_buddy_block *
++__get_buddy(struct gpu_buddy_block *block)
++{
++ struct gpu_buddy_block *parent;
++
++ parent = block->parent;
++ if (!parent)
++ return NULL;
++
++ if (parent->left == block)
++ return parent->right;
++
++ return parent->left;
++}
++
++static unsigned int __gpu_buddy_free(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block,
++ bool force_merge)
++{
++ struct gpu_buddy_block *parent;
++ unsigned int order;
++
++ while ((parent = block->parent)) {
++ struct gpu_buddy_block *buddy;
++
++ buddy = __get_buddy(block);
++
++ if (!gpu_buddy_block_is_free(buddy))
++ break;
++
++ if (!force_merge) {
++ /*
++ * Check the block and its buddy clear state and exit
++ * the loop if they both have the dissimilar state.
++ */
++ if (gpu_buddy_block_is_clear(block) !=
++ gpu_buddy_block_is_clear(buddy))
++ break;
++
++ if (gpu_buddy_block_is_clear(block))
++ mark_cleared(parent);
++ }
++
++ rbtree_remove(mm, buddy);
++ if (force_merge && gpu_buddy_block_is_clear(buddy))
++ mm->clear_avail -= gpu_buddy_block_size(mm, buddy);
++
++ gpu_block_free(mm, block);
++ gpu_block_free(mm, buddy);
++
++ block = parent;
++ }
++
++ order = gpu_buddy_block_order(block);
++ mark_free(mm, block);
++
++ return order;
++}
++
++static int __force_merge(struct gpu_buddy *mm,
++ u64 start,
++ u64 end,
++ unsigned int min_order)
++{
++ unsigned int tree, order;
++ int i;
++
++ if (!min_order)
++ return -ENOMEM;
++
++ if (min_order > mm->max_order)
++ return -EINVAL;
++
++ for_each_free_tree(tree) {
++ for (i = min_order - 1; i >= 0; i--) {
++ struct rb_node *iter = rb_last(&mm->free_trees[tree][i]);
++
++ while (iter) {
++ struct gpu_buddy_block *block, *buddy;
++ u64 block_start, block_end;
++
++ block = rbtree_get_free_block(iter);
++ iter = rb_prev(iter);
++
++ if (!block || !block->parent)
++ continue;
++
++ block_start = gpu_buddy_block_offset(block);
++ block_end = block_start + gpu_buddy_block_size(mm, block) - 1;
++
++ if (!contains(start, end, block_start, block_end))
++ continue;
++
++ buddy = __get_buddy(block);
++ if (!gpu_buddy_block_is_free(buddy))
++ continue;
++
++ WARN_ON(gpu_buddy_block_is_clear(block) ==
++ gpu_buddy_block_is_clear(buddy));
++
++ /*
++ * Advance to the next node when the current node is the buddy,
++ * as freeing the block will also remove its buddy from the tree.
++ */
++ if (iter == &buddy->rb)
++ iter = rb_prev(iter);
++
++ rbtree_remove(mm, block);
++ if (gpu_buddy_block_is_clear(block))
++ mm->clear_avail -= gpu_buddy_block_size(mm, block);
++
++ order = __gpu_buddy_free(mm, block, true);
++ if (order >= min_order)
++ return 0;
++ }
++ }
++ }
++
++ return -ENOMEM;
++}
++
++/**
++ * gpu_buddy_init - init memory manager
++ *
++ * @mm: GPU buddy manager to initialize
++ * @size: size in bytes to manage
++ * @chunk_size: minimum page size in bytes for our allocations
++ *
++ * Initializes the memory manager and its resources.
++ *
++ * Returns:
++ * 0 on success, error code on failure.
++ */
++int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
++{
++ unsigned int i, j, root_count = 0;
++ u64 offset = 0;
++
++ if (size < chunk_size)
++ return -EINVAL;
++
++ if (chunk_size < SZ_4K)
++ return -EINVAL;
++
++ if (!is_power_of_2(chunk_size))
++ return -EINVAL;
++
++ size = round_down(size, chunk_size);
++
++ mm->size = size;
++ mm->avail = size;
++ mm->clear_avail = 0;
++ mm->chunk_size = chunk_size;
++ mm->max_order = ilog2(size) - ilog2(chunk_size);
++
++ BUG_ON(mm->max_order > GPU_BUDDY_MAX_ORDER);
++
++ mm->free_trees = kmalloc_array(GPU_BUDDY_MAX_FREE_TREES,
++ sizeof(*mm->free_trees),
++ GFP_KERNEL);
++ if (!mm->free_trees)
++ return -ENOMEM;
++
++ for_each_free_tree(i) {
++ mm->free_trees[i] = kmalloc_array(mm->max_order + 1,
++ sizeof(struct rb_root),
++ GFP_KERNEL);
++ if (!mm->free_trees[i])
++ goto out_free_tree;
++
++ for (j = 0; j <= mm->max_order; ++j)
++ mm->free_trees[i][j] = RB_ROOT;
++ }
++
++ mm->n_roots = hweight64(size);
++
++ mm->roots = kmalloc_array(mm->n_roots,
++ sizeof(struct gpu_buddy_block *),
++ GFP_KERNEL);
++ if (!mm->roots)
++ goto out_free_tree;
++
++ /*
++ * Split into power-of-two blocks, in case we are given a size that is
++ * not itself a power-of-two.
++ */
++ do {
++ struct gpu_buddy_block *root;
++ unsigned int order;
++ u64 root_size;
++
++ order = ilog2(size) - ilog2(chunk_size);
++ root_size = chunk_size << order;
++
++ root = gpu_block_alloc(mm, NULL, order, offset);
++ if (!root)
++ goto out_free_roots;
++
++ mark_free(mm, root);
++
++ BUG_ON(root_count > mm->max_order);
++ BUG_ON(gpu_buddy_block_size(mm, root) < chunk_size);
++
++ mm->roots[root_count] = root;
++
++ offset += root_size;
++ size -= root_size;
++ root_count++;
++ } while (size);
++
++ return 0;
++
++out_free_roots:
++ while (root_count--)
++ gpu_block_free(mm, mm->roots[root_count]);
++ kfree(mm->roots);
++out_free_tree:
++ while (i--)
++ kfree(mm->free_trees[i]);
++ kfree(mm->free_trees);
++ return -ENOMEM;
++}
++EXPORT_SYMBOL(gpu_buddy_init);
++
++/**
++ * gpu_buddy_fini - tear down the memory manager
++ *
++ * @mm: GPU buddy manager to free
++ *
++ * Cleanup memory manager resources and the freetree
++ */
++void gpu_buddy_fini(struct gpu_buddy *mm)
++{
++ u64 root_size, size, start;
++ unsigned int order;
++ int i;
++
++ size = mm->size;
++
++ for (i = 0; i < mm->n_roots; ++i) {
++ order = ilog2(size) - ilog2(mm->chunk_size);
++ start = gpu_buddy_block_offset(mm->roots[i]);
++ __force_merge(mm, start, start + size, order);
++
++ if (WARN_ON(!gpu_buddy_block_is_free(mm->roots[i])))
++ kunit_fail_current_test("buddy_fini() root");
++
++ gpu_block_free(mm, mm->roots[i]);
++
++ root_size = mm->chunk_size << order;
++ size -= root_size;
++ }
++
++ WARN_ON(mm->avail != mm->size);
++
++ for_each_free_tree(i)
++ kfree(mm->free_trees[i]);
++ kfree(mm->free_trees);
++ kfree(mm->roots);
++}
++EXPORT_SYMBOL(gpu_buddy_fini);
++
++static int split_block(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ unsigned int block_order = gpu_buddy_block_order(block) - 1;
++ u64 offset = gpu_buddy_block_offset(block);
++
++ BUG_ON(!gpu_buddy_block_is_free(block));
++ BUG_ON(!gpu_buddy_block_order(block));
++
++ block->left = gpu_block_alloc(mm, block, block_order, offset);
++ if (!block->left)
++ return -ENOMEM;
++
++ block->right = gpu_block_alloc(mm, block, block_order,
++ offset + (mm->chunk_size << block_order));
++ if (!block->right) {
++ gpu_block_free(mm, block->left);
++ return -ENOMEM;
++ }
++
++ mark_split(mm, block);
++
++ if (gpu_buddy_block_is_clear(block)) {
++ mark_cleared(block->left);
++ mark_cleared(block->right);
++ clear_reset(block);
++ }
++
++ mark_free(mm, block->left);
++ mark_free(mm, block->right);
++
++ return 0;
++}
++
++/**
++ * gpu_get_buddy - get buddy address
++ *
++ * @block: GPU buddy block
++ *
++ * Returns the corresponding buddy block for @block, or NULL
++ * if this is a root block and can't be merged further.
++ * Requires some kind of locking to protect against
++ * any concurrent allocate and free operations.
++ */
++struct gpu_buddy_block *
++gpu_get_buddy(struct gpu_buddy_block *block)
++{
++ return __get_buddy(block);
++}
++EXPORT_SYMBOL(gpu_get_buddy);
++
++/**
++ * gpu_buddy_reset_clear - reset blocks clear state
++ *
++ * @mm: GPU buddy manager
++ * @is_clear: blocks clear state
++ *
++ * Reset the clear state based on @is_clear value for each block
++ * in the freetree.
++ */
++void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear)
++{
++ enum gpu_buddy_free_tree src_tree, dst_tree;
++ u64 root_size, size, start;
++ unsigned int order;
++ int i;
++
++ size = mm->size;
++ for (i = 0; i < mm->n_roots; ++i) {
++ order = ilog2(size) - ilog2(mm->chunk_size);
++ start = gpu_buddy_block_offset(mm->roots[i]);
++ __force_merge(mm, start, start + size, order);
++
++ root_size = mm->chunk_size << order;
++ size -= root_size;
++ }
++
++ src_tree = is_clear ? GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
++ dst_tree = is_clear ? GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
++
++ for (i = 0; i <= mm->max_order; ++i) {
++ struct rb_root *root = &mm->free_trees[src_tree][i];
++ struct gpu_buddy_block *block, *tmp;
++
++ rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
++ rbtree_remove(mm, block);
++ if (is_clear) {
++ mark_cleared(block);
++ mm->clear_avail += gpu_buddy_block_size(mm, block);
++ } else {
++ clear_reset(block);
++ mm->clear_avail -= gpu_buddy_block_size(mm, block);
++ }
++
++ rbtree_insert(mm, block, dst_tree);
++ }
++ }
++}
++EXPORT_SYMBOL(gpu_buddy_reset_clear);
++
++/**
++ * gpu_buddy_free_block - free a block
++ *
++ * @mm: GPU buddy manager
++ * @block: block to be freed
++ */
++void gpu_buddy_free_block(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ BUG_ON(!gpu_buddy_block_is_allocated(block));
++ mm->avail += gpu_buddy_block_size(mm, block);
++ if (gpu_buddy_block_is_clear(block))
++ mm->clear_avail += gpu_buddy_block_size(mm, block);
++
++ __gpu_buddy_free(mm, block, false);
++}
++EXPORT_SYMBOL(gpu_buddy_free_block);
++
++static void __gpu_buddy_free_list(struct gpu_buddy *mm,
++ struct list_head *objects,
++ bool mark_clear,
++ bool mark_dirty)
++{
++ struct gpu_buddy_block *block, *on;
++
++ WARN_ON(mark_dirty && mark_clear);
++
++ list_for_each_entry_safe(block, on, objects, link) {
++ if (mark_clear)
++ mark_cleared(block);
++ else if (mark_dirty)
++ clear_reset(block);
++ gpu_buddy_free_block(mm, block);
++ cond_resched();
++ }
++ INIT_LIST_HEAD(objects);
++}
++
++static void gpu_buddy_free_list_internal(struct gpu_buddy *mm,
++ struct list_head *objects)
++{
++ /*
++ * Don't touch the clear/dirty bit, since allocation is still internal
++ * at this point. For example we might have just failed part of the
++ * allocation.
++ */
++ __gpu_buddy_free_list(mm, objects, false, false);
++}
++
++/**
++ * gpu_buddy_free_list - free blocks
++ *
++ * @mm: GPU buddy manager
++ * @objects: input list head to free blocks
++ * @flags: optional flags like GPU_BUDDY_CLEARED
++ */
++void gpu_buddy_free_list(struct gpu_buddy *mm,
++ struct list_head *objects,
++ unsigned int flags)
++{
++ bool mark_clear = flags & GPU_BUDDY_CLEARED;
++
++ __gpu_buddy_free_list(mm, objects, mark_clear, !mark_clear);
++}
++EXPORT_SYMBOL(gpu_buddy_free_list);
++
++static bool block_incompatible(struct gpu_buddy_block *block, unsigned int flags)
++{
++ bool needs_clear = flags & GPU_BUDDY_CLEAR_ALLOCATION;
++
++ return needs_clear != gpu_buddy_block_is_clear(block);
++}
++
++static struct gpu_buddy_block *
++__alloc_range_bias(struct gpu_buddy *mm,
++ u64 start, u64 end,
++ unsigned int order,
++ unsigned long flags,
++ bool fallback)
++{
++ u64 req_size = mm->chunk_size << order;
++ struct gpu_buddy_block *block;
++ struct gpu_buddy_block *buddy;
++ LIST_HEAD(dfs);
++ int err;
++ int i;
++
++ end = end - 1;
++
++ for (i = 0; i < mm->n_roots; ++i)
++ list_add_tail(&mm->roots[i]->tmp_link, &dfs);
++
++ do {
++ u64 block_start;
++ u64 block_end;
++
++ block = list_first_entry_or_null(&dfs,
++ struct gpu_buddy_block,
++ tmp_link);
++ if (!block)
++ break;
++
++ list_del(&block->tmp_link);
++
++ if (gpu_buddy_block_order(block) < order)
++ continue;
++
++ block_start = gpu_buddy_block_offset(block);
++ block_end = block_start + gpu_buddy_block_size(mm, block) - 1;
++
++ if (!overlaps(start, end, block_start, block_end))
++ continue;
++
++ if (gpu_buddy_block_is_allocated(block))
++ continue;
++
++ if (block_start < start || block_end > end) {
++ u64 adjusted_start = max(block_start, start);
++ u64 adjusted_end = min(block_end, end);
++
++ if (round_down(adjusted_end + 1, req_size) <=
++ round_up(adjusted_start, req_size))
++ continue;
++ }
++
++ if (!fallback && block_incompatible(block, flags))
++ continue;
++
++ if (contains(start, end, block_start, block_end) &&
++ order == gpu_buddy_block_order(block)) {
++ /*
++ * Find the free block within the range.
++ */
++ if (gpu_buddy_block_is_free(block))
++ return block;
++
++ continue;
++ }
++
++ if (!gpu_buddy_block_is_split(block)) {
++ err = split_block(mm, block);
++ if (unlikely(err))
++ goto err_undo;
++ }
++
++ list_add(&block->right->tmp_link, &dfs);
++ list_add(&block->left->tmp_link, &dfs);
++ } while (1);
++
++ return ERR_PTR(-ENOSPC);
++
++err_undo:
++ /*
++ * We really don't want to leave around a bunch of split blocks, since
++ * bigger is better, so make sure we merge everything back before we
++ * free the allocated blocks.
++ */
++ buddy = __get_buddy(block);
++ if (buddy &&
++ (gpu_buddy_block_is_free(block) &&
++ gpu_buddy_block_is_free(buddy)))
++ __gpu_buddy_free(mm, block, false);
++ return ERR_PTR(err);
++}
++
++static struct gpu_buddy_block *
++__gpu_buddy_alloc_range_bias(struct gpu_buddy *mm,
++ u64 start, u64 end,
++ unsigned int order,
++ unsigned long flags)
++{
++ struct gpu_buddy_block *block;
++ bool fallback = false;
++
++ block = __alloc_range_bias(mm, start, end, order,
++ flags, fallback);
++ if (IS_ERR(block))
++ return __alloc_range_bias(mm, start, end, order,
++ flags, !fallback);
++
++ return block;
++}
++
++static struct gpu_buddy_block *
++get_maxblock(struct gpu_buddy *mm,
++ unsigned int order,
++ enum gpu_buddy_free_tree tree)
++{
++ struct gpu_buddy_block *max_block = NULL, *block = NULL;
++ struct rb_root *root;
++ unsigned int i;
++
++ for (i = order; i <= mm->max_order; ++i) {
++ root = &mm->free_trees[tree][i];
++ block = rbtree_last_free_block(root);
++ if (!block)
++ continue;
++
++ if (!max_block) {
++ max_block = block;
++ continue;
++ }
++
++ if (gpu_buddy_block_offset(block) >
++ gpu_buddy_block_offset(max_block)) {
++ max_block = block;
++ }
++ }
++
++ return max_block;
++}
++
++static struct gpu_buddy_block *
++alloc_from_freetree(struct gpu_buddy *mm,
++ unsigned int order,
++ unsigned long flags)
++{
++ struct gpu_buddy_block *block = NULL;
++ struct rb_root *root;
++ enum gpu_buddy_free_tree tree;
++ unsigned int tmp;
++ int err;
++
++ tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ?
++ GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
++
++ if (flags & GPU_BUDDY_TOPDOWN_ALLOCATION) {
++ block = get_maxblock(mm, order, tree);
++ if (block)
++ /* Store the obtained block order */
++ tmp = gpu_buddy_block_order(block);
++ } else {
++ for (tmp = order; tmp <= mm->max_order; ++tmp) {
++ /* Get RB tree root for this order and tree */
++ root = &mm->free_trees[tree][tmp];
++ block = rbtree_last_free_block(root);
++ if (block)
++ break;
++ }
++ }
++
++ if (!block) {
++ /* Try allocating from the other tree */
++ tree = (tree == GPU_BUDDY_CLEAR_TREE) ?
++ GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
++
++ for (tmp = order; tmp <= mm->max_order; ++tmp) {
++ root = &mm->free_trees[tree][tmp];
++ block = rbtree_last_free_block(root);
++ if (block)
++ break;
++ }
++
++ if (!block)
++ return ERR_PTR(-ENOSPC);
++ }
++
++ BUG_ON(!gpu_buddy_block_is_free(block));
++
++ while (tmp != order) {
++ err = split_block(mm, block);
++ if (unlikely(err))
++ goto err_undo;
++
++ block = block->right;
++ tmp--;
++ }
++ return block;
++
++err_undo:
++ if (tmp != order)
++ __gpu_buddy_free(mm, block, false);
++ return ERR_PTR(err);
++}
++
++static int __alloc_range(struct gpu_buddy *mm,
++ struct list_head *dfs,
++ u64 start, u64 size,
++ struct list_head *blocks,
++ u64 *total_allocated_on_err)
++{
++ struct gpu_buddy_block *block;
++ struct gpu_buddy_block *buddy;
++ u64 total_allocated = 0;
++ LIST_HEAD(allocated);
++ u64 end;
++ int err;
++
++ end = start + size - 1;
++
++ do {
++ u64 block_start;
++ u64 block_end;
++
++ block = list_first_entry_or_null(dfs,
++ struct gpu_buddy_block,
++ tmp_link);
++ if (!block)
++ break;
++
++ list_del(&block->tmp_link);
++
++ block_start = gpu_buddy_block_offset(block);
++ block_end = block_start + gpu_buddy_block_size(mm, block) - 1;
++
++ if (!overlaps(start, end, block_start, block_end))
++ continue;
++
++ if (gpu_buddy_block_is_allocated(block)) {
++ err = -ENOSPC;
++ goto err_free;
++ }
++
++ if (contains(start, end, block_start, block_end)) {
++ if (gpu_buddy_block_is_free(block)) {
++ mark_allocated(mm, block);
++ total_allocated += gpu_buddy_block_size(mm, block);
++ mm->avail -= gpu_buddy_block_size(mm, block);
++ if (gpu_buddy_block_is_clear(block))
++ mm->clear_avail -= gpu_buddy_block_size(mm, block);
++ list_add_tail(&block->link, &allocated);
++ continue;
++ } else if (!mm->clear_avail) {
++ err = -ENOSPC;
++ goto err_free;
++ }
++ }
++
++ if (!gpu_buddy_block_is_split(block)) {
++ err = split_block(mm, block);
++ if (unlikely(err))
++ goto err_undo;
++ }
++
++ list_add(&block->right->tmp_link, dfs);
++ list_add(&block->left->tmp_link, dfs);
++ } while (1);
++
++ if (total_allocated < size) {
++ err = -ENOSPC;
++ goto err_free;
++ }
++
++ list_splice_tail(&allocated, blocks);
++
++ return 0;
++
++err_undo:
++ /*
++ * We really don't want to leave around a bunch of split blocks, since
++ * bigger is better, so make sure we merge everything back before we
++ * free the allocated blocks.
++ */
++ buddy = __get_buddy(block);
++ if (buddy &&
++ (gpu_buddy_block_is_free(block) &&
++ gpu_buddy_block_is_free(buddy)))
++ __gpu_buddy_free(mm, block, false);
++
++err_free:
++ if (err == -ENOSPC && total_allocated_on_err) {
++ list_splice_tail(&allocated, blocks);
++ *total_allocated_on_err = total_allocated;
++ } else {
++ gpu_buddy_free_list_internal(mm, &allocated);
++ }
++
++ return err;
++}
++
++static int __gpu_buddy_alloc_range(struct gpu_buddy *mm,
++ u64 start,
++ u64 size,
++ u64 *total_allocated_on_err,
++ struct list_head *blocks)
++{
++ LIST_HEAD(dfs);
++ int i;
++
++ for (i = 0; i < mm->n_roots; ++i)
++ list_add_tail(&mm->roots[i]->tmp_link, &dfs);
++
++ return __alloc_range(mm, &dfs, start, size,
++ blocks, total_allocated_on_err);
++}
++
++static int __alloc_contig_aligned_retry(struct gpu_buddy *mm,
++ u64 unaligned_offset,
++ u64 size,
++ u64 min_block_size,
++ struct list_head *blocks)
++{
++ u64 aligned_offset = round_down(unaligned_offset, min_block_size);
++
++ return __gpu_buddy_alloc_range(mm, aligned_offset, size, NULL, blocks);
++}
++
++static int __alloc_contig_try_harder(struct gpu_buddy *mm,
++ u64 size,
++ u64 min_block_size,
++ struct list_head *blocks)
++{
++ u64 rhs_offset, lhs_offset, filled;
++ struct gpu_buddy_block *block;
++ unsigned int tree, order;
++ u64 modify_size;
++ int err;
++
++ modify_size = rounddown_pow_of_two(size);
++ order = ilog2(modify_size) - ilog2(mm->chunk_size);
++ if (order == 0)
++ return -ENOSPC;
++
++ for_each_free_tree(tree) {
++ struct rb_root *root;
++ struct rb_node *iter;
++
++ root = &mm->free_trees[tree][order];
++ if (rbtree_is_empty(root))
++ continue;
++
++ iter = rb_last(root);
++ while (iter) {
++ block = rbtree_get_free_block(iter);
++
++ rhs_offset = gpu_buddy_block_offset(block);
++
++ /* Allocate blocks traversing RHS */
++ err = __gpu_buddy_alloc_range(mm, rhs_offset, size,
++ &filled, blocks);
++ if (err && err != -ENOSPC)
++ return err;
++ if (!err && IS_ALIGNED(rhs_offset, min_block_size))
++ return 0;
++ if (!err) {
++ /* Allocate the unaligned RHS offset using round_down */
++ gpu_buddy_free_list_internal(mm, blocks);
++ err = __alloc_contig_aligned_retry(mm, rhs_offset,
++ size,
++ min_block_size,
++ blocks);
++ if (!err)
++ return 0;
++ if (err != -ENOSPC) {
++ gpu_buddy_free_list_internal(mm, blocks);
++ return err;
++ }
++ goto next;
++ }
++
++ if (size - filled > rhs_offset)
++ goto next;
++
++ lhs_offset = rhs_offset - (size - filled);
++
++ /* Allocate the unaligned LHS offset using round_down */
++ gpu_buddy_free_list_internal(mm, blocks);
++ err = __alloc_contig_aligned_retry(mm, lhs_offset, size,
++ min_block_size, blocks);
++ if (!err)
++ return 0;
++ if (err != -ENOSPC) {
++ gpu_buddy_free_list_internal(mm, blocks);
++ return err;
++ }
++next:
++ gpu_buddy_free_list_internal(mm, blocks);
++ iter = rb_prev(iter);
++ }
++ }
++
++ return -ENOSPC;
++}
++
++/**
++ * gpu_buddy_block_trim - free unused pages
++ *
++ * @mm: GPU buddy manager
++ * @start: start address to begin the trimming.
++ * @new_size: original size requested
++ * @blocks: Input and output list of allocated blocks.
++ * MUST contain single block as input to be trimmed.
++ * On success will contain the newly allocated blocks
++ * making up the @new_size. Blocks always appear in
++ * ascending order
++ *
++ * For contiguous allocation, we round up the size to the nearest
++ * power of two value, drivers consume *actual* size, so remaining
++ * portions are unused and can be optionally freed with this function
++ *
++ * Returns:
++ * 0 on success, error code on failure.
++ */
++int gpu_buddy_block_trim(struct gpu_buddy *mm,
++ u64 *start,
++ u64 new_size,
++ struct list_head *blocks)
++{
++ struct gpu_buddy_block *parent;
++ struct gpu_buddy_block *block;
++ u64 block_start, block_end;
++ LIST_HEAD(dfs);
++ u64 new_start;
++ int err;
++
++ if (!list_is_singular(blocks))
++ return -EINVAL;
++
++ block = list_first_entry(blocks,
++ struct gpu_buddy_block,
++ link);
++
++ block_start = gpu_buddy_block_offset(block);
++ block_end = block_start + gpu_buddy_block_size(mm, block);
++
++ if (WARN_ON(!gpu_buddy_block_is_allocated(block)))
++ return -EINVAL;
++
++ if (new_size > gpu_buddy_block_size(mm, block))
++ return -EINVAL;
++
++ if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size))
++ return -EINVAL;
++
++ if (new_size == gpu_buddy_block_size(mm, block))
++ return 0;
++
++ new_start = block_start;
++ if (start) {
++ new_start = *start;
++
++ if (new_start < block_start)
++ return -EINVAL;
++
++ if (!IS_ALIGNED(new_start, mm->chunk_size))
++ return -EINVAL;
++
++ if (range_overflows(new_start, new_size, block_end))
++ return -EINVAL;
++ }
++
++ list_del(&block->link);
++ mark_free(mm, block);
++ mm->avail += gpu_buddy_block_size(mm, block);
++ if (gpu_buddy_block_is_clear(block))
++ mm->clear_avail += gpu_buddy_block_size(mm, block);
++
++ /* Prevent recursively freeing this node */
++ parent = block->parent;
++ block->parent = NULL;
++
++ list_add(&block->tmp_link, &dfs);
++ err = __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL);
++ if (err) {
++ mark_allocated(mm, block);
++ mm->avail -= gpu_buddy_block_size(mm, block);
++ if (gpu_buddy_block_is_clear(block))
++ mm->clear_avail -= gpu_buddy_block_size(mm, block);
++ list_add(&block->link, blocks);
++ }
++
++ block->parent = parent;
++ return err;
++}
++EXPORT_SYMBOL(gpu_buddy_block_trim);
++
++static struct gpu_buddy_block *
++__gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
++ u64 start, u64 end,
++ unsigned int order,
++ unsigned long flags)
++{
++ if (flags & GPU_BUDDY_RANGE_ALLOCATION)
++ /* Allocate traversing within the range */
++ return __gpu_buddy_alloc_range_bias(mm, start, end,
++ order, flags);
++ else
++ /* Allocate from freetree */
++ return alloc_from_freetree(mm, order, flags);
++}
++
++/**
++ * gpu_buddy_alloc_blocks - allocate power-of-two blocks
++ *
++ * @mm: GPU buddy manager to allocate from
++ * @start: start of the allowed range for this block
++ * @end: end of the allowed range for this block
++ * @size: size of the allocation in bytes
++ * @min_block_size: alignment of the allocation
++ * @blocks: output list head to add allocated blocks
++ * @flags: GPU_BUDDY_*_ALLOCATION flags
++ *
++ * alloc_range_bias() called on range limitations, which traverses
++ * the tree and returns the desired block.
++ *
++ * alloc_from_freetree() called when *no* range restrictions
++ * are enforced, which picks the block from the freetree.
++ *
++ * Returns:
++ * 0 on success, error code on failure.
++ */
++int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
++ u64 start, u64 end, u64 size,
++ u64 min_block_size,
++ struct list_head *blocks,
++ unsigned long flags)
++{
++ struct gpu_buddy_block *block = NULL;
++ u64 original_size, original_min_size;
++ unsigned int min_order, order;
++ LIST_HEAD(allocated);
++ unsigned long pages;
++ int err;
++
++ if (size < mm->chunk_size)
++ return -EINVAL;
++
++ if (min_block_size < mm->chunk_size)
++ return -EINVAL;
++
++ if (!is_power_of_2(min_block_size))
++ return -EINVAL;
++
++ if (!IS_ALIGNED(start | end | size, mm->chunk_size))
++ return -EINVAL;
++
++ if (end > mm->size)
++ return -EINVAL;
++
++ if (range_overflows(start, size, mm->size))
++ return -EINVAL;
++
++ /* Actual range allocation */
++ if (start + size == end) {
++ if (!IS_ALIGNED(start | end, min_block_size))
++ return -EINVAL;
++
++ return __gpu_buddy_alloc_range(mm, start, size, NULL, blocks);
++ }
++
++ original_size = size;
++ original_min_size = min_block_size;
++
++ /* Roundup the size to power of 2 */
++ if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) {
++ size = roundup_pow_of_two(size);
++ min_block_size = size;
++ /* Align size value to min_block_size */
++ } else if (!IS_ALIGNED(size, min_block_size)) {
++ size = round_up(size, min_block_size);
++ }
++
++ pages = size >> ilog2(mm->chunk_size);
++ order = fls(pages) - 1;
++ min_order = ilog2(min_block_size) - ilog2(mm->chunk_size);
++
++ if (order > mm->max_order || size > mm->size) {
++ if ((flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) &&
++ !(flags & GPU_BUDDY_RANGE_ALLOCATION))
++ return __alloc_contig_try_harder(mm, original_size,
++ original_min_size, blocks);
++
++ return -EINVAL;
++ }
++
++ do {
++ order = min(order, (unsigned int)fls(pages) - 1);
++ BUG_ON(order > mm->max_order);
++ BUG_ON(order < min_order);
++
++ do {
++ block = __gpu_buddy_alloc_blocks(mm, start,
++ end,
++ order,
++ flags);
++ if (!IS_ERR(block))
++ break;
++
++ if (order-- == min_order) {
++ /* Try allocation through force merge method */
++ if (mm->clear_avail &&
++ !__force_merge(mm, start, end, min_order)) {
++ block = __gpu_buddy_alloc_blocks(mm, start,
++ end,
++ min_order,
++ flags);
++ if (!IS_ERR(block)) {
++ order = min_order;
++ break;
++ }
++ }
++
++ /*
++ * Try contiguous block allocation through
++ * try harder method.
++ */
++ if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION &&
++ !(flags & GPU_BUDDY_RANGE_ALLOCATION))
++ return __alloc_contig_try_harder(mm,
++ original_size,
++ original_min_size,
++ blocks);
++ err = -ENOSPC;
++ goto err_free;
++ }
++ } while (1);
++
++ mark_allocated(mm, block);
++ mm->avail -= gpu_buddy_block_size(mm, block);
++ if (gpu_buddy_block_is_clear(block))
++ mm->clear_avail -= gpu_buddy_block_size(mm, block);
++ kmemleak_update_trace(block);
++ list_add_tail(&block->link, &allocated);
++
++ pages -= BIT(order);
++
++ if (!pages)
++ break;
++ } while (1);
++
++ /* Trim the allocated block to the required size */
++ if (!(flags & GPU_BUDDY_TRIM_DISABLE) &&
++ original_size != size) {
++ struct list_head *trim_list;
++ LIST_HEAD(temp);
++ u64 trim_size;
++
++ trim_list = &allocated;
++ trim_size = original_size;
++
++ if (!list_is_singular(&allocated)) {
++ block = list_last_entry(&allocated, typeof(*block), link);
++ list_move(&block->link, &temp);
++ trim_list = &temp;
++ trim_size = gpu_buddy_block_size(mm, block) -
++ (size - original_size);
++ }
++
++ gpu_buddy_block_trim(mm,
++ NULL,
++ trim_size,
++ trim_list);
++
++ if (!list_empty(&temp))
++ list_splice_tail(trim_list, &allocated);
++ }
++
++ list_splice_tail(&allocated, blocks);
++ return 0;
++
++err_free:
++ gpu_buddy_free_list_internal(mm, &allocated);
++ return err;
++}
++EXPORT_SYMBOL(gpu_buddy_alloc_blocks);
++
++/**
++ * gpu_buddy_block_print - print block information
++ *
++ * @mm: GPU buddy manager
++ * @block: GPU buddy block
++ */
++void gpu_buddy_block_print(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block)
++{
++ u64 start = gpu_buddy_block_offset(block);
++ u64 size = gpu_buddy_block_size(mm, block);
++
++ pr_info("%#018llx-%#018llx: %llu\n", start, start + size, size);
++}
++EXPORT_SYMBOL(gpu_buddy_block_print);
++
++/**
++ * gpu_buddy_print - print allocator state
++ *
++ * @mm: GPU buddy manager
++ * @p: GPU printer to use
++ */
++void gpu_buddy_print(struct gpu_buddy *mm)
++{
++ int order;
++
++ pr_info("chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n",
++ mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
++
++ for (order = mm->max_order; order >= 0; order--) {
++ struct gpu_buddy_block *block, *tmp;
++ struct rb_root *root;
++ u64 count = 0, free;
++ unsigned int tree;
++
++ for_each_free_tree(tree) {
++ root = &mm->free_trees[tree][order];
++
++ rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
++ BUG_ON(!gpu_buddy_block_is_free(block));
++ count++;
++ }
++ }
++
++ free = count * (mm->chunk_size << order);
++ if (free < SZ_1M)
++ pr_info("order-%2d free: %8llu KiB, blocks: %llu\n",
++ order, free >> 10, count);
++ else
++ pr_info("order-%2d free: %8llu MiB, blocks: %llu\n",
++ order, free >> 20, count);
++ }
++}
++EXPORT_SYMBOL(gpu_buddy_print);
++
++static void gpu_buddy_module_exit(void)
++{
++ kmem_cache_destroy(slab_blocks);
++}
++
++static int __init gpu_buddy_module_init(void)
++{
++ slab_blocks = KMEM_CACHE(gpu_buddy_block, 0);
++ if (!slab_blocks)
++ return -ENOMEM;
++
++ return 0;
++}
++
++module_init(gpu_buddy_module_init);
++module_exit(gpu_buddy_module_exit);
++
++MODULE_DESCRIPTION("GPU Buddy Allocator");
++MODULE_LICENSE("Dual MIT/GPL");
+diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
+index 0dd0d996e53e9a..62b9fdae96ddbf 100644
+--- a/drivers/gpu/drm/Kconfig
++++ b/drivers/gpu/drm/Kconfig
+@@ -85,7 +85,6 @@ config DRM_KUNIT_TEST
+ select DRM_EXPORT_FOR_TESTS if m
+ select DRM_GEM_SHMEM_HELPER
+ select DRM_KUNIT_TEST_HELPERS
+- select DRM_LIB_RANDOM
+ select PRIME_NUMBERS
+ default KUNIT_ALL_TESTS
+ help
+@@ -325,6 +324,7 @@ config DRM_GPUVM
+ config DRM_BUDDY
+ tristate
+ depends on DRM
++ select GPU_BUDDY
+ help
+ A page based buddy allocator
+
+@@ -507,10 +507,6 @@ config DRM_EXPORT_FOR_TESTS
+ config DRM_PANEL_BACKLIGHT_QUIRKS
+ tristate
+
+-config DRM_LIB_RANDOM
+- bool
+- default n
+-
+ config DRM_PRIVACY_SCREEN
+ bool
+ default n
+diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
+index f4a5edf746d2d6..4a899e75a25cf8 100644
+--- a/drivers/gpu/drm/Makefile
++++ b/drivers/gpu/drm/Makefile
+@@ -75,7 +75,6 @@ drm-y := \
+ drm_vblank_work.o \
+ drm_vma_manager.o \
+ drm_writeback.o
+-drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o
+ drm-$(CONFIG_COMPAT) += drm_ioc32.o
+ drm-$(CONFIG_DRM_PANEL) += drm_panel.o
+ drm-$(CONFIG_OF) += drm_of.o
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+index c4ca5987567922..f252432245bfa8 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+@@ -559,7 +559,9 @@ static int acp_hw_fini(void *handle)
+
+ mfd_remove_devices(adev->acp.parent);
+ kfree(adev->acp.acp_res);
++ pm_genpd_remove(&adev->acp.acp_genpd->gpd);
+ kfree(adev->acp.acp_genpd);
++ adev->acp.acp_genpd = NULL;
+ kfree(adev->acp.acp_cell);
+
+ return 0;
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+index a3a7d20ab4fea9..94e05c6b39d442 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+@@ -345,6 +345,45 @@ static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev)
+ }
+
+ #ifdef CONFIG_ACPI
++/**
++ * amdgpu_acpi_vfct_match() - Check if a VFCT entry matches the device
++ * @adev: AMDGPU device
++ * @vhdr: VFCT image header to check
++ *
++ * VFCT entries contain the PCI bus number as recorded during BIOS POST.
++ * On systems where the kernel renumbers PCI buses (e.g. pci=realloc or
++ * resource conflicts), the runtime bus number may differ from the POST
++ * value. Match by device identity (vendor + device + function) and use
++ * the bus number as a preference: exact bus match is preferred, but when
++ * the bus numbers disagree we accept the entry if the device identity
++ * matches.
++ *
++ * Returns: 0 on match, -ENODEV on no match
++ */
++static int amdgpu_acpi_vfct_match(struct amdgpu_device *adev,
++ VFCT_IMAGE_HEADER *vhdr)
++{
++ /* Vendor and device IDs must always match */
++ if (vhdr->VendorID != adev->pdev->vendor ||
++ vhdr->DeviceID != adev->pdev->device)
++ return -ENODEV;
++
++ if (vhdr->PCIDevice != PCI_SLOT(adev->pdev->devfn) ||
++ vhdr->PCIFunction != PCI_FUNC(adev->pdev->devfn))
++ return -ENODEV;
++
++ /* Exact bus number match - preferred */
++ if (vhdr->PCIBus == adev->pdev->bus->number)
++ return 0;
++
++ /* Bus mismatch but device identity matches (PCI renumbering case) */
++ dev_notice(adev->dev,
++ "VFCT bus number mismatch: table %u != runtime %u, matching by device identity (vendor 0x%04x device 0x%04x)\n",
++ vhdr->PCIBus, adev->pdev->bus->number,
++ adev->pdev->vendor, adev->pdev->device);
++ return 0;
++}
++
+ static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
+ {
+ struct acpi_table_header *hdr;
+@@ -380,11 +419,7 @@ static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
+ }
+
+ if (vhdr->ImageLength &&
+- vhdr->PCIBus == adev->pdev->bus->number &&
+- vhdr->PCIDevice == PCI_SLOT(adev->pdev->devfn) &&
+- vhdr->PCIFunction == PCI_FUNC(adev->pdev->devfn) &&
+- vhdr->VendorID == adev->pdev->vendor &&
+- vhdr->DeviceID == adev->pdev->device) {
++ !amdgpu_acpi_vfct_match(adev, vhdr)) {
+ adev->bios = kmemdup(&vbios->VbiosContent,
+ vhdr->ImageLength,
+ GFP_KERNEL);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+index e018a807e4e3ae..028af05dec6c87 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+@@ -272,13 +272,17 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
+ goto free_partial_kdata;
+ break;
+
++ case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
++ if (size < sizeof(struct drm_amdgpu_cs_chunk_cp_gfx_shadow))
++ goto free_partial_kdata;
++ break;
++
+ case AMDGPU_CHUNK_ID_DEPENDENCIES:
+ case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
+ case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
+ case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
+ case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
+ case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
+- case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
+ break;
+
+ default:
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+index d1daf5a79bbd40..df77afbd54629a 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+@@ -1757,6 +1757,15 @@ static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device
+
+ if (c->x86_vendor == X86_VENDOR_INTEL)
+ return false;
++
++ /*
++ * AMD Ryzen Pinnacle Ridge (Zen+, family 0x17 model 0x08) CPUs don't
++ * support PCIe dynamic speed switching.
++ * https://gitlab.freedesktop.org/drm/amd/-/work_items/5436
++ */
++ if (c->x86_vendor == X86_VENDOR_AMD && c->x86 == 0x17 &&
++ c->x86_model == 0x08)
++ return false;
+ #endif
+ return true;
+ }
+@@ -4626,8 +4635,6 @@ static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
+
+ iounmap(adev->rmmio);
+ adev->rmmio = NULL;
+- if (adev->mman.aper_base_kaddr)
+- iounmap(adev->mman.aper_base_kaddr);
+ adev->mman.aper_base_kaddr = NULL;
+
+ /* Memory manager related */
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+index fc588ef598c094..3d7c6d0dd39427 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+@@ -273,10 +273,12 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
+ goto error_free;
+ }
+
+- r = amdgpu_bo_pin(*bo_ptr, domain);
+- if (r) {
+- dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
+- goto error_unreserve;
++ if (free) {
++ r = amdgpu_bo_pin(*bo_ptr, domain);
++ if (r) {
++ dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
++ goto error_unreserve;
++ }
+ }
+
+ r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
+@@ -299,7 +301,8 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
+ return 0;
+
+ error_unpin:
+- amdgpu_bo_unpin(*bo_ptr);
++ if (free)
++ amdgpu_bo_unpin(*bo_ptr);
+ error_unreserve:
+ amdgpu_bo_unreserve(*bo_ptr);
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
+index 50fcd86e1033b8..aee50e7fb4208e 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
+@@ -55,7 +55,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
+ uint64_t start, uint64_t size,
+ struct amdgpu_res_cursor *cur)
+ {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ struct list_head *head, *next;
+ struct drm_mm_node *node;
+
+@@ -71,7 +71,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
+ head = &to_amdgpu_vram_mgr_resource(res)->blocks;
+
+ block = list_first_entry_or_null(head,
+- struct drm_buddy_block,
++ struct gpu_buddy_block,
+ link);
+ if (!block)
+ goto fallback;
+@@ -81,7 +81,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
+
+ next = block->link.next;
+ if (next != head)
+- block = list_entry(next, struct drm_buddy_block, link);
++ block = list_entry(next, struct gpu_buddy_block, link);
+ }
+
+ cur->start = amdgpu_vram_mgr_block_start(block) + start;
+@@ -124,7 +124,7 @@ fallback:
+ */
+ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
+ {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ struct drm_mm_node *node;
+ struct list_head *next;
+
+@@ -145,7 +145,7 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
+ block = cur->node;
+
+ next = block->link.next;
+- block = list_entry(next, struct drm_buddy_block, link);
++ block = list_entry(next, struct gpu_buddy_block, link);
+
+ cur->node = block;
+ cur->start = amdgpu_vram_mgr_block_start(block);
+@@ -173,7 +173,7 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
+ */
+ static inline bool amdgpu_res_cleared(struct amdgpu_res_cursor *cur)
+ {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+
+ switch (cur->mem_type) {
+ case TTM_PL_VRAM:
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+index 40dd04a4f7df9e..13d97d5a094bce 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+@@ -501,6 +501,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
+
+ if (new_mem->mem_type == TTM_PL_TT ||
+ new_mem->mem_type == AMDGPU_PL_PREEMPT) {
++ if (old_mem && (old_mem->mem_type == TTM_PL_TT ||
++ old_mem->mem_type == AMDGPU_PL_PREEMPT)) {
++ r = ttm_bo_wait_ctx(bo, ctx);
++ if (r)
++ return r;
++
++ amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm);
++ }
++
+ r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem);
+ if (r)
+ return r;
+@@ -535,6 +544,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
+ ttm_bo_assign_mem(bo, new_mem);
+ return 0;
+ }
++ if ((old_mem->mem_type == TTM_PL_TT ||
++ old_mem->mem_type == AMDGPU_PL_PREEMPT) &&
++ (new_mem->mem_type == TTM_PL_TT ||
++ new_mem->mem_type == AMDGPU_PL_PREEMPT)) {
++ amdgpu_bo_move_notify(bo, evict, new_mem);
++ ttm_resource_free(bo, &bo->resource);
++ ttm_bo_assign_mem(bo, new_mem);
++ return 0;
++ }
+
+ if (old_mem->mem_type == AMDGPU_PL_GDS ||
+ old_mem->mem_type == AMDGPU_PL_GWS ||
+@@ -1889,18 +1907,23 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
+ /* Change the size here instead of the init above so only lpfn is affected */
+ amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ #ifdef CONFIG_64BIT
+-#ifdef CONFIG_X86
+- if (adev->gmc.xgmi.connected_to_cpu)
+- adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base,
+- adev->gmc.visible_vram_size);
+-
+- else if (adev->gmc.is_app_apu)
++ if (adev->gmc.xgmi.connected_to_cpu) {
++ void *kaddr = devm_memremap(adev->dev, adev->gmc.aper_base,
++ adev->gmc.visible_vram_size,
++ MEMREMAP_WB);
++ if (IS_ERR(kaddr))
++ return PTR_ERR(kaddr);
++ adev->mman.aper_base_kaddr = (__force void __iomem *)kaddr;
++ } else if (adev->gmc.is_app_apu) {
+ DRM_DEBUG_DRIVER(
+ "No need to ioremap when real vram size is 0\n");
+- else
+-#endif
+- adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
+- adev->gmc.visible_vram_size);
++ } else {
++ adev->mman.aper_base_kaddr = devm_ioremap_wc(adev->dev,
++ adev->gmc.aper_base,
++ adev->gmc.visible_vram_size);
++ if (!adev->mman.aper_base_kaddr)
++ return -ENOMEM;
++ }
+ #endif
+
+ /*
+@@ -2035,8 +2058,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
+ */
+ void amdgpu_ttm_fini(struct amdgpu_device *adev)
+ {
+- int idx;
+-
+ if (!adev->mman.initialized)
+ return;
+
+@@ -2059,14 +2080,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
+ amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
+
+- if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+-
+- if (adev->mman.aper_base_kaddr)
+- iounmap(adev->mman.aper_base_kaddr);
+- adev->mman.aper_base_kaddr = NULL;
+-
+- drm_dev_exit(idx);
+- }
++ adev->mman.aper_base_kaddr = NULL;
+
+ amdgpu_vram_mgr_fini(adev);
+ amdgpu_gtt_mgr_fini(adev);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+index 65bb26215e867a..779dcd820d4a93 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+@@ -135,7 +135,7 @@ MODULE_FIRMWARE(FIRMWARE_VEGA12);
+ MODULE_FIRMWARE(FIRMWARE_VEGA20);
+
+ static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
+-static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo);
++static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *abo);
+
+ static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev,
+ uint32_t size,
+@@ -158,7 +158,7 @@ static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev,
+ amdgpu_bo_kunmap(bo);
+ amdgpu_bo_unpin(bo);
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
+- amdgpu_uvd_force_into_uvd_segment(bo);
++ amdgpu_uvd_force_into_vcpu_segment(bo);
+ r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+ if (r)
+ goto err;
+@@ -188,6 +188,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
+ const struct common_firmware_header *hdr;
+ unsigned int family_id;
+ int i, j, r;
++ u32 vcpu_bo_domain;
+
+ INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler);
+
+@@ -319,12 +320,20 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
+ if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
+ bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
+
++ /* UVD 5.0 and newer HW can use 64 bit addressing. */
++ adev->uvd.address_64_bit =
++ !amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0);
++
++ vcpu_bo_domain = AMDGPU_GEM_DOMAIN_VRAM;
++ if (adev->uvd.address_64_bit)
++ vcpu_bo_domain |= AMDGPU_GEM_DOMAIN_GTT;
++
+ for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+ if (adev->uvd.harvest_config & (1 << j))
+ continue;
++
+ r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
+- AMDGPU_GEM_DOMAIN_VRAM |
+- AMDGPU_GEM_DOMAIN_GTT,
++ vcpu_bo_domain,
+ &adev->uvd.inst[j].vcpu_bo,
+ &adev->uvd.inst[j].gpu_addr,
+ &adev->uvd.inst[j].cpu_addr);
+@@ -339,10 +348,6 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
+ adev->uvd.filp[i] = NULL;
+ }
+
+- /* from uvd v5.0 HW addressing capacity increased to 64 bits */
+- if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0))
+- adev->uvd.address_64_bit = true;
+-
+ r = amdgpu_uvd_create_msg_bo_helper(adev, 128 << 10, &adev->uvd.ib_bo);
+ if (r)
+ return r;
+@@ -544,6 +549,24 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
+ }
+ }
+
++static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *bo)
++{
++ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
++ struct amdgpu_bo *vcpu_bo = adev->uvd.inst[0].vcpu_bo;
++ struct amdgpu_res_cursor vcpu_cur;
++
++ amdgpu_res_first(vcpu_bo->tbo.resource, 0,
++ amdgpu_bo_size(vcpu_bo), &vcpu_cur);
++
++ bo->placement.num_placement = 1;
++ bo->placement.placement = &bo->placements[0];
++ bo->placements[0].fpfn = ALIGN_DOWN(vcpu_cur.start, SZ_256M) >> PAGE_SHIFT;
++ bo->placements[0].lpfn = bo->placements[0].fpfn + (SZ_256M >> PAGE_SHIFT);
++ bo->placements[0].mem_type = vcpu_bo->tbo.resource->mem_type;
++ if (bo->placements[0].mem_type == TTM_PL_VRAM)
++ bo->placements[0].flags |= TTM_PL_FLAG_CONTIGUOUS;
++}
++
+ static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo)
+ {
+ int i;
+@@ -594,13 +617,10 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
+ if (!ctx->parser->adev->uvd.address_64_bit) {
+ /* check if it's a message or feedback command */
+ cmd = amdgpu_ib_get_value(ctx->ib, ctx->idx) >> 1;
+- if (cmd == 0x0 || cmd == 0x3) {
+- /* yes, force it into VRAM */
+- uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM;
+-
+- amdgpu_bo_placement_from_domain(bo, domain);
+- }
+- amdgpu_uvd_force_into_uvd_segment(bo);
++ if (cmd == 0x0 || cmd == 0x3)
++ amdgpu_uvd_force_into_vcpu_segment(bo);
++ else
++ amdgpu_uvd_force_into_uvd_segment(bo);
+
+ r = ttm_bo_validate(&bo->tbo, &bo->placement, &tctx);
+ }
+@@ -634,6 +654,14 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg,
+ unsigned int image_size, tmp, min_dpb_size, num_dpb_buffer;
+ unsigned int min_ctx_size = ~0;
+
++ /* Reject invalid dimensions to prevent division by zero */
++ if (width < 16 || height < 16) {
++ dev_WARN_ONCE(adev->dev, 1,
++ "Invalid UVD decoding dimensions (%dx%d)!\n",
++ width, height);
++ return -EINVAL;
++ }
++
+ image_size = width * height;
+ image_size += image_size / 2;
+ image_size = ALIGN(image_size, 1024);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+index fa89c69b750a7e..33b4305bd305ce 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+@@ -852,9 +852,20 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p,
+ goto out;
+ }
+
+- *size = amdgpu_ib_get_value(ib, idx + 8) *
+- amdgpu_ib_get_value(ib, idx + 10) *
+- 8 * 3 / 2;
++ uint32_t width, height;
++ width = amdgpu_ib_get_value(ib, idx + 8);
++ height = amdgpu_ib_get_value(ib, idx + 10);
++
++ if (width == 0 || height == 0 ||
++ width > 4096 || height > 2304) {
++ DRM_ERROR("invalid VCE image size: %ux%u\n",
++ width, height);
++ r = -EINVAL;
++ goto out;
++ }
++
++ *size = width * height * 8 * 3 / 2;
++
+ break;
+
+ case 0x04000001: /* config extension */
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+index a29d01202b5b3f..a6b34e5ba96ce2 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+@@ -2308,19 +2308,6 @@ static void amdgpu_vm_destroy_task_info(struct kref *kref)
+ kfree(ti);
+ }
+
+-static inline struct amdgpu_vm *
+-amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid)
+-{
+- struct amdgpu_vm *vm;
+- unsigned long flags;
+-
+- xa_lock_irqsave(&adev->vm_manager.pasids, flags);
+- vm = xa_load(&adev->vm_manager.pasids, pasid);
+- xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
+-
+- return vm;
+-}
+-
+ /**
+ * amdgpu_vm_put_task_info - reference down the vm task_info ptr
+ *
+@@ -2366,8 +2353,16 @@ amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm)
+ struct amdgpu_task_info *
+ amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid)
+ {
+- return amdgpu_vm_get_task_info_vm(
+- amdgpu_vm_get_vm_from_pasid(adev, pasid));
++ struct amdgpu_task_info *ti;
++ struct amdgpu_vm *vm;
++ unsigned long flags;
++
++ xa_lock_irqsave(&adev->vm_manager.pasids, flags);
++ vm = xa_load(&adev->vm_manager.pasids, pasid);
++ ti = amdgpu_vm_get_task_info_vm(vm);
++ xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
++
++ return ti;
+ }
+
+ static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm)
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+index 54f2e7b3927964..76ddfc7325a37a 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+@@ -24,6 +24,7 @@
+
+ #include <linux/dma-mapping.h>
+ #include <drm/ttm/ttm_range_manager.h>
++#include <drm/drm_buddy.h>
+
+ #include "amdgpu.h"
+ #include "amdgpu_vm.h"
+@@ -52,15 +53,15 @@ to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
+ return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
+ }
+
+-static inline struct drm_buddy_block *
++static inline struct gpu_buddy_block *
+ amdgpu_vram_mgr_first_block(struct list_head *list)
+ {
+- return list_first_entry_or_null(list, struct drm_buddy_block, link);
++ return list_first_entry_or_null(list, struct gpu_buddy_block, link);
+ }
+
+ static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
+ {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ u64 start, size;
+
+ block = amdgpu_vram_mgr_first_block(head);
+@@ -71,7 +72,7 @@ static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
+ start = amdgpu_vram_mgr_block_start(block);
+ size = amdgpu_vram_mgr_block_size(block);
+
+- block = list_entry(block->link.next, struct drm_buddy_block, link);
++ block = list_entry(block->link.next, struct gpu_buddy_block, link);
+ if (start + size != amdgpu_vram_mgr_block_start(block))
+ return false;
+ }
+@@ -81,7 +82,7 @@ static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
+
+ static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head)
+ {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ u64 size = 0;
+
+ list_for_each_entry(block, head, link)
+@@ -254,7 +255,7 @@ const struct attribute_group amdgpu_vram_mgr_attr_group = {
+ * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM
+ */
+ static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
+- struct drm_buddy_block *block)
++ struct gpu_buddy_block *block)
+ {
+ u64 start = amdgpu_vram_mgr_block_start(block);
+ u64 end = start + amdgpu_vram_mgr_block_size(block);
+@@ -279,7 +280,7 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
+ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+ struct ttm_resource *res = bo->tbo.resource;
+ struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ u64 usage = 0;
+
+ if (amdgpu_gmc_vram_full_visible(&adev->gmc))
+@@ -299,15 +300,15 @@ static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
+ {
+ struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
+ struct amdgpu_device *adev = to_amdgpu_device(mgr);
+- struct drm_buddy *mm = &mgr->mm;
++ struct gpu_buddy *mm = &mgr->mm;
+ struct amdgpu_vram_reservation *rsv, *temp;
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ uint64_t vis_usage;
+
+ list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {
+- if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
++ if (gpu_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
+ rsv->size, mm->chunk_size, &rsv->allocated,
+- DRM_BUDDY_RANGE_ALLOCATION))
++ GPU_BUDDY_RANGE_ALLOCATION))
+ continue;
+
+ block = amdgpu_vram_mgr_first_block(&rsv->allocated);
+@@ -460,8 +461,8 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+ struct amdgpu_vram_mgr_resource *vres;
+ u64 size, remaining_size, lpfn, fpfn;
+ unsigned int adjust_dcc_size = 0;
+- struct drm_buddy *mm = &mgr->mm;
+- struct drm_buddy_block *block;
++ struct gpu_buddy *mm = &mgr->mm;
++ struct gpu_buddy_block *block;
+ unsigned long pages_per_block;
+ int r;
+
+@@ -503,17 +504,17 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+ INIT_LIST_HEAD(&vres->blocks);
+
+ if (place->flags & TTM_PL_FLAG_TOPDOWN)
+- vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
++ vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
+
+ if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
+- vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
++ vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
+
+ if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED)
+- vres->flags |= DRM_BUDDY_CLEAR_ALLOCATION;
++ vres->flags |= GPU_BUDDY_CLEAR_ALLOCATION;
+
+ if (fpfn || lpfn != mgr->mm.size)
+ /* Allocate blocks in desired range */
+- vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
++ vres->flags |= GPU_BUDDY_RANGE_ALLOCATION;
+
+ if (bo->flags & AMDGPU_GEM_CREATE_GFX12_DCC &&
+ adev->gmc.gmc_funcs->get_dcc_alignment)
+@@ -526,7 +527,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+ dcc_size = roundup_pow_of_two(vres->base.size + adjust_dcc_size);
+ remaining_size = (u64)dcc_size;
+
+- vres->flags |= DRM_BUDDY_TRIM_DISABLE;
++ vres->flags |= GPU_BUDDY_TRIM_DISABLE;
+ }
+
+ mutex_lock(&mgr->lock);
+@@ -546,7 +547,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+
+ BUG_ON(min_block_size < mm->chunk_size);
+
+- r = drm_buddy_alloc_blocks(mm, fpfn,
++ r = gpu_buddy_alloc_blocks(mm, fpfn,
+ lpfn,
+ size,
+ min_block_size,
+@@ -555,7 +556,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+
+ if (unlikely(r == -ENOSPC) && pages_per_block == ~0ul &&
+ !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) {
+- vres->flags &= ~DRM_BUDDY_CONTIGUOUS_ALLOCATION;
++ vres->flags &= ~GPU_BUDDY_CONTIGUOUS_ALLOCATION;
+ pages_per_block = max_t(u32, 2UL << (20UL - PAGE_SHIFT),
+ tbo->page_alignment);
+
+@@ -572,7 +573,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+ }
+
+ if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) {
+- struct drm_buddy_block *dcc_block;
++ struct gpu_buddy_block *dcc_block;
+ unsigned long dcc_start;
+ u64 trim_start;
+
+@@ -582,7 +583,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+ roundup((unsigned long)amdgpu_vram_mgr_block_start(dcc_block),
+ adjust_dcc_size);
+ trim_start = (u64)dcc_start;
+- drm_buddy_block_trim(mm, &trim_start,
++ gpu_buddy_block_trim(mm, &trim_start,
+ (u64)vres->base.size,
+ &vres->blocks);
+ }
+@@ -620,7 +621,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
+ return 0;
+
+ error_free_blocks:
+- drm_buddy_free_list(mm, &vres->blocks, 0);
++ gpu_buddy_free_list(mm, &vres->blocks, 0);
+ mutex_unlock(&mgr->lock);
+ error_fini:
+ ttm_resource_fini(man, &vres->base);
+@@ -643,15 +644,15 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
+ struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
+ struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
+ struct amdgpu_device *adev = to_amdgpu_device(mgr);
+- struct drm_buddy *mm = &mgr->mm;
+- struct drm_buddy_block *block;
++ struct gpu_buddy *mm = &mgr->mm;
++ struct gpu_buddy_block *block;
+ uint64_t vis_usage = 0;
+
+ mutex_lock(&mgr->lock);
+ list_for_each_entry(block, &vres->blocks, link)
+ vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
+
+- drm_buddy_free_list(mm, &vres->blocks, vres->flags);
++ gpu_buddy_free_list(mm, &vres->blocks, vres->flags);
+ amdgpu_vram_mgr_do_reserve(man);
+ mutex_unlock(&mgr->lock);
+
+@@ -690,7 +691,7 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
+ if (!*sgt)
+ return -ENOMEM;
+
+- /* Determine the number of DRM_BUDDY blocks to export */
++ /* Determine the number of GPU_BUDDY blocks to export */
+ amdgpu_res_first(res, offset, length, &cursor);
+ while (cursor.remaining) {
+ num_entries++;
+@@ -706,10 +707,10 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
+ sg->length = 0;
+
+ /*
+- * Walk down DRM_BUDDY blocks to populate scatterlist nodes
+- * @note: Use iterator api to get first the DRM_BUDDY block
++ * Walk down GPU_BUDDY blocks to populate scatterlist nodes
++ * @note: Use iterator api to get first the GPU_BUDDY block
+ * and the number of bytes from it. Access the following
+- * DRM_BUDDY block(s) if more buffer needs to exported
++ * GPU_BUDDY block(s) if more buffer needs to exported
+ */
+ amdgpu_res_first(res, offset, length, &cursor);
+ for_each_sgtable_sg((*sgt), sg, i) {
+@@ -794,10 +795,10 @@ uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
+ void amdgpu_vram_mgr_clear_reset_blocks(struct amdgpu_device *adev)
+ {
+ struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
+- struct drm_buddy *mm = &mgr->mm;
++ struct gpu_buddy *mm = &mgr->mm;
+
+ mutex_lock(&mgr->lock);
+- drm_buddy_reset_clear(mm, false);
++ gpu_buddy_reset_clear(mm, false);
+ mutex_unlock(&mgr->lock);
+ }
+
+@@ -817,7 +818,7 @@ static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,
+ size_t size)
+ {
+ struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+
+ /* Check each drm buddy block individually */
+ list_for_each_entry(block, &mgr->blocks, link) {
+@@ -850,7 +851,7 @@ static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,
+ size_t size)
+ {
+ struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+
+ /* Check each drm buddy block individually */
+ list_for_each_entry(block, &mgr->blocks, link) {
+@@ -879,7 +880,7 @@ static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
+ struct drm_printer *printer)
+ {
+ struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
+- struct drm_buddy *mm = &mgr->mm;
++ struct gpu_buddy *mm = &mgr->mm;
+ struct amdgpu_vram_reservation *rsv;
+
+ drm_printf(printer, " vis usage:%llu\n",
+@@ -938,7 +939,7 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
+ if (!adev->gmc.is_app_apu) {
+ man->func = &amdgpu_vram_mgr_func;
+
+- err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
++ err = gpu_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
+ if (err)
+ return err;
+ } else {
+@@ -977,11 +978,11 @@ void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
+ kfree(rsv);
+
+ list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
+- drm_buddy_free_list(&mgr->mm, &rsv->allocated, 0);
++ gpu_buddy_free_list(&mgr->mm, &rsv->allocated, 0);
+ kfree(rsv);
+ }
+ if (!adev->gmc.is_app_apu)
+- drm_buddy_fini(&mgr->mm);
++ gpu_buddy_fini(&mgr->mm);
+ mutex_unlock(&mgr->lock);
+
+ ttm_resource_manager_cleanup(man);
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
+index b256cbc2bc2708..3afa344a2f5384 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
+@@ -24,11 +24,11 @@
+ #ifndef __AMDGPU_VRAM_MGR_H__
+ #define __AMDGPU_VRAM_MGR_H__
+
+-#include <drm/drm_buddy.h>
++#include <linux/gpu_buddy.h>
+
+ struct amdgpu_vram_mgr {
+ struct ttm_resource_manager manager;
+- struct drm_buddy mm;
++ struct gpu_buddy mm;
+ /* protects access to buffer objects */
+ struct mutex lock;
+ struct list_head reservations_pending;
+@@ -43,19 +43,19 @@ struct amdgpu_vram_mgr_resource {
+ unsigned long flags;
+ };
+
+-static inline u64 amdgpu_vram_mgr_block_start(struct drm_buddy_block *block)
++static inline u64 amdgpu_vram_mgr_block_start(struct gpu_buddy_block *block)
+ {
+- return drm_buddy_block_offset(block);
++ return gpu_buddy_block_offset(block);
+ }
+
+-static inline u64 amdgpu_vram_mgr_block_size(struct drm_buddy_block *block)
++static inline u64 amdgpu_vram_mgr_block_size(struct gpu_buddy_block *block)
+ {
+- return (u64)PAGE_SIZE << drm_buddy_block_order(block);
++ return (u64)PAGE_SIZE << gpu_buddy_block_order(block);
+ }
+
+-static inline bool amdgpu_vram_mgr_is_cleared(struct drm_buddy_block *block)
++static inline bool amdgpu_vram_mgr_is_cleared(struct gpu_buddy_block *block)
+ {
+- return drm_buddy_block_is_clear(block);
++ return gpu_buddy_block_is_clear(block);
+ }
+
+ static inline struct amdgpu_vram_mgr_resource *
+@@ -66,7 +66,7 @@ to_amdgpu_vram_mgr_resource(struct ttm_resource *res)
+
+ static inline void amdgpu_vram_mgr_set_cleared(struct ttm_resource *res)
+ {
+- to_amdgpu_vram_mgr_resource(res)->flags |= DRM_BUDDY_CLEARED;
++ to_amdgpu_vram_mgr_resource(res)->flags |= GPU_BUDDY_CLEARED;
+ }
+
+ #endif
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+index a6e22c897b9d34..698a14cd1b8d60 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+@@ -3934,7 +3934,7 @@ static void gfx_v10_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
+ WAIT_REG_MEM_ENGINE(eng_sel)));
+
+ if (mem_space)
+- BUG_ON(addr0 & 0x3); /* Dword align */
++ WARN_ON(addr0 & 0x3); /* Dword align */
+ amdgpu_ring_write(ring, addr0);
+ amdgpu_ring_write(ring, addr1);
+ amdgpu_ring_write(ring, ref);
+@@ -5195,6 +5195,15 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev)
+ gfx_v10_0_get_tcc_info(adev);
+ adev->gfx.config.pa_sc_tile_steering_override =
+ gfx_v10_0_init_pa_sc_tile_steering_override(adev);
++ /* Program DB_RING_CONTROL for multiple GFX pipes
++ * Default power up value is 1.
++ * Possible values:
++ * 0 - split occlusion counters between gfx pipes
++ * 1 - all occlusion counters to pipe 0
++ * 2 - all occlusion counters to pipe 1
++ */
++ WREG32_FIELD15(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL,
++ (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1);
+
+ /* XXX SH_MEM regs */
+ /* where to put LDS, scratch, GPUVM in FSA64 space */
+@@ -8556,7 +8565,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
+ }
+
+ amdgpu_ring_write(ring, header);
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -8591,7 +8600,7 @@ static void gfx_v10_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -8624,9 +8633,9 @@ static void gfx_v10_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ * aligned if only send 32bit data low (discard data high)
+ */
+ if (write64bit)
+- BUG_ON(addr & 0x7);
++ WARN_ON(addr & 0x7);
+ else
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -8674,9 +8683,6 @@ static void gfx_v10_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+ {
+ struct amdgpu_device *adev = ring->adev;
+
+- /* we only allocate 32bit for each seq wb address */
+- BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+-
+ /* write fence seq to the "addr" */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+index 6b5b5fcdb988ca..d530dc183c76b7 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+@@ -496,7 +496,7 @@ static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
+ WAIT_REG_MEM_ENGINE(eng_sel)));
+
+ if (mem_space)
+- BUG_ON(addr0 & 0x3); /* Dword align */
++ WARN_ON(addr0 & 0x3); /* Dword align */
+ amdgpu_ring_write(ring, addr0);
+ amdgpu_ring_write(ring, addr1);
+ amdgpu_ring_write(ring, ref);
+@@ -5673,7 +5673,7 @@ static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
+ control |= 0x400000;
+
+ amdgpu_ring_write(ring, header);
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -5712,7 +5712,7 @@ static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -5745,9 +5745,9 @@ static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ * aligned if only send 32bit data low (discard data high)
+ */
+ if (write64bit)
+- BUG_ON(addr & 0x7);
++ WARN_ON(addr & 0x7);
+ else
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -5805,9 +5805,6 @@ static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+ {
+ struct amdgpu_device *adev = ring->adev;
+
+- /* we only allocate 32bit for each seq wb address */
+- BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+-
+ /* write fence seq to the "addr" */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+index 97116a92cd8b3a..3a6d6ee76918e3 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+@@ -409,7 +409,7 @@ static void gfx_v12_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
+ WAIT_REG_MEM_ENGINE(eng_sel)));
+
+ if (mem_space)
+- BUG_ON(addr0 & 0x3); /* Dword align */
++ WARN_ON(addr0 & 0x3); /* Dword align */
+ amdgpu_ring_write(ring, addr0);
+ amdgpu_ring_write(ring, addr1);
+ amdgpu_ring_write(ring, ref);
+@@ -4338,7 +4338,7 @@ static void gfx_v12_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
+ control |= 0x400000;
+
+ amdgpu_ring_write(ring, header);
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -4361,7 +4361,7 @@ static void gfx_v12_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
+ control |= 0x40000000;
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -4392,9 +4392,9 @@ static void gfx_v12_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ * aligned if only send 32bit data low (discard data high)
+ */
+ if (write64bit)
+- BUG_ON(addr & 0x7);
++ WARN_ON(addr & 0x7);
+ else
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -4446,9 +4446,6 @@ static void gfx_v12_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+ {
+ struct amdgpu_device *adev = ring->adev;
+
+- /* we only allocate 32bit for each seq wb address */
+- BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+-
+ /* write fence seq to the "addr" */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+index 2e54fb63dd5bc0..aecf2b466f8bf1 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+@@ -6272,9 +6272,6 @@ static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
+ static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+ u64 seq, unsigned int flags)
+ {
+- /* we only allocate 32bit for each seq wb address */
+- BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+-
+ /* write fence seq to the "addr" */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+index d16ac8669e07bd..06ab40412200fa 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+@@ -1156,7 +1156,7 @@ static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
+ WAIT_REG_MEM_ENGINE(eng_sel)));
+
+ if (mem_space)
+- BUG_ON(addr0 & 0x3); /* Dword align */
++ WARN_ON(addr0 & 0x3); /* Dword align */
+ amdgpu_ring_write(ring, addr0);
+ amdgpu_ring_write(ring, addr1);
+ amdgpu_ring_write(ring, ref);
+@@ -5415,7 +5415,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
+ }
+
+ amdgpu_ring_write(ring, header);
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -5527,7 +5527,7 @@ static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -5568,9 +5568,9 @@ static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ * aligned if only send 32bit data low (discard data high)
+ */
+ if (write64bit)
+- BUG_ON(addr & 0x7);
++ WARN_ON(addr & 0x7);
+ else
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+index 15d482990297cc..07dd7c2102e095 100644
+--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
++++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+@@ -409,7 +409,7 @@ static void gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
+ WAIT_REG_MEM_ENGINE(eng_sel)));
+
+ if (mem_space)
+- BUG_ON(addr0 & 0x3); /* Dword align */
++ WARN_ON(addr0 & 0x3); /* Dword align */
+ amdgpu_ring_write(ring, addr0);
+ amdgpu_ring_write(ring, addr1);
+ amdgpu_ring_write(ring, ref);
+@@ -2882,7 +2882,7 @@ static void gfx_v9_4_3_ring_emit_ib_compute(struct amdgpu_ring *ring,
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+- BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
++ WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+ #ifdef __BIG_ENDIAN
+ (2 << 0) |
+@@ -2916,9 +2916,9 @@ static void gfx_v9_4_3_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ * aligned if only send 32bit data low (discard data high)
+ */
+ if (write64bit)
+- BUG_ON(addr & 0x7);
++ WARN_ON(addr & 0x7);
+ else
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -2978,9 +2978,6 @@ static void gfx_v9_4_3_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+ {
+ struct amdgpu_device *adev = ring->adev;
+
+- /* we only allocate 32bit for each seq wb address */
+- BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+-
+ /* write fence seq to the "addr" */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+index 8c97a67f4c9f0d..29d3985e4effe9 100644
+--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
++++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+@@ -450,7 +450,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64
+ /* write the fence */
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -460,7 +460,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64
+ addr += 4;
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(seq));
+diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+index 34fcbcdd93b6cf..2f575d7e297c41 100644
+--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+@@ -557,7 +557,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -568,7 +568,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(seq));
+diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+index 241995252ff0bb..1d3ca139065a66 100644
+--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
++++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+@@ -376,7 +376,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -387,7 +387,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(seq));
+diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+index 74e79ddd714acf..2e6adbd54a5ee3 100644
+--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+@@ -357,7 +357,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -368,7 +368,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(seq));
+diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+index 1d0131c172d61b..58aa2eda2ca2d9 100644
+--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+@@ -389,7 +389,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+@@ -400,7 +400,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
+ amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
+ /* zero in first two bits */
+- BUG_ON(addr & 0x3);
++ WARN_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(seq));
+diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+index 041531a799d466..80373143949dcf 100644
+--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
++++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+@@ -1868,14 +1868,17 @@ out:
+ #define RENCODE_IB_PARAM_SESSION_INIT 0x00000003
+
+ /* return the offset in ib if id is found, -1 otherwise */
+-static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start)
++static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start, uint32_t *length)
+ {
+ int i;
+ uint32_t len;
+
+ for (i = start; (len = amdgpu_ib_get_value(ib, i)) >= 8; i += len / 4) {
+- if (amdgpu_ib_get_value(ib, i + 1) == id)
++ if (amdgpu_ib_get_value(ib, i + 1) == id) {
++ if (length)
++ *length = len;
+ return i;
++ }
+ }
+ return -1;
+ }
+@@ -1885,14 +1888,14 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
+ struct amdgpu_ib *ib)
+ {
+ struct amdgpu_ring *ring = amdgpu_job_ring(job);
+- uint32_t val;
++ uint32_t val, len;
+ int idx = 0, sidx;
+
+ /* The first instance can decode anything */
+ if (!ring->me)
+ return 0;
+
+- while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx)) >= 0) {
++ while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx, &len)) >= 0) {
+ val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */
+ if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
+ uint32_t valid_buf_flag = amdgpu_ib_get_value(ib, idx + 6);
+@@ -1905,12 +1908,12 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
+ amdgpu_ib_get_value(ib, idx + 8);
+ return vcn_v4_0_dec_msg(p, job, msg_buffer_addr);
+ } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) {
+- sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx);
++ sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx, NULL);
+ if (sidx >= 0 &&
+ amdgpu_ib_get_value(ib, sidx + 2) == RENCODE_ENCODE_STANDARD_AV1)
+ return vcn_v4_0_limit_sched(p, job);
+ }
+- idx += amdgpu_ib_get_value(ib, idx) / 4;
++ idx += len / 4;
+ }
+ return 0;
+ }
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+index e75ca4dfc6a9ac..8366709a72116f 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+@@ -1795,13 +1795,13 @@ static int criu_checkpoint_devices(struct kfd_process *p,
+ struct kfd_criu_device_bucket *device_buckets = NULL;
+ int ret = 0, i;
+
+- device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL);
++ device_buckets = kvcalloc(num_devices, sizeof(*device_buckets), GFP_KERNEL);
+ if (!device_buckets) {
+ ret = -ENOMEM;
+ goto exit;
+ }
+
+- device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL);
++ device_priv = kvcalloc(num_devices, sizeof(*device_priv), GFP_KERNEL);
+ if (!device_priv) {
+ ret = -ENOMEM;
+ goto exit;
+@@ -1921,17 +1921,17 @@ static int criu_checkpoint_bos(struct kfd_process *p,
+ int ret = 0, pdd_index, bo_index = 0, id;
+ void *mem;
+
+- bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL);
++ bo_buckets = kvcalloc(num_bos, sizeof(*bo_buckets), GFP_KERNEL);
+ if (!bo_buckets)
+ return -ENOMEM;
+
+- bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL);
++ bo_privs = kvcalloc(num_bos, sizeof(*bo_privs), GFP_KERNEL);
+ if (!bo_privs) {
+ ret = -ENOMEM;
+ goto exit;
+ }
+
+- files = kvzalloc(num_bos * sizeof(struct file *), GFP_KERNEL);
++ files = kvcalloc(num_bos, sizeof(struct file *), GFP_KERNEL);
+ if (!files) {
+ ret = -ENOMEM;
+ goto exit;
+@@ -2470,7 +2470,7 @@ static int criu_restore_bos(struct kfd_process *p,
+ if (!bo_buckets)
+ return -ENOMEM;
+
+- files = kvzalloc(args->num_bos * sizeof(struct file *), GFP_KERNEL);
++ files = kvcalloc(args->num_bos, sizeof(struct file *), GFP_KERNEL);
+ if (!files) {
+ ret = -ENOMEM;
+ goto exit;
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+index 0b5f1e574afeee..05c049c6d3b52c 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+@@ -107,6 +107,9 @@ static int allocate_event_notification_slot(struct kfd_process *p,
+ }
+
+ if (restore_id) {
++ if (*restore_id >= KFD_SIGNAL_EVENT_LIMIT)
++ return -EINVAL;
++
+ id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,
+ GFP_KERNEL);
+ } else {
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+index 1f1169a4d15408..de2af34865fbda 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+@@ -23,6 +23,7 @@
+ */
+
+ #include <linux/slab.h>
++#include <linux/overflow.h>
+ #include "kfd_priv.h"
+ #include "kfd_topology.h"
+ #include "kfd_svm.h"
+@@ -235,7 +236,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope
+ struct kfd_topology_device *topo_dev;
+ u64 expected_queue_size;
+ struct amdgpu_vm *vm;
+- u32 total_cwsr_size;
++ u64 total_cwsr_size;
+ int err;
+
+ topo_dev = kfd_topology_device_by_id(pdd->dev->id);
+@@ -305,8 +306,14 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope
+ goto out_err_unreserve;
+ }
+
+- total_cwsr_size = (properties->ctx_save_restore_area_size +
+- topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask);
++ total_cwsr_size = (u64)properties->ctx_save_restore_area_size +
++ topo_dev->node_props.debug_memory_size;
++ if (check_mul_overflow(total_cwsr_size,
++ NUM_XCC(pdd->dev->xcc_mask),
++ &total_cwsr_size)) {
++ err = -EINVAL;
++ goto out_err_unreserve;
++ }
+ total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);
+
+ err = kfd_queue_buffer_get(vm, (void *)properties->ctx_save_restore_area_address,
+@@ -341,7 +348,7 @@ out_err_release:
+ int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)
+ {
+ struct kfd_topology_device *topo_dev;
+- u32 total_cwsr_size;
++ u64 total_cwsr_size;
+
+ kfd_queue_buffer_put(&properties->wptr_bo);
+ kfd_queue_buffer_put(&properties->rptr_bo);
+@@ -352,8 +359,12 @@ int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_prope
+ topo_dev = kfd_topology_device_by_id(pdd->dev->id);
+ if (!topo_dev)
+ return -EINVAL;
+- total_cwsr_size = (properties->ctx_save_restore_area_size +
+- topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask);
++ total_cwsr_size = (u64)properties->ctx_save_restore_area_size +
++ topo_dev->node_props.debug_memory_size;
++ if (check_mul_overflow(total_cwsr_size,
++ NUM_XCC(pdd->dev->xcc_mask),
++ &total_cwsr_size))
++ return -EINVAL;
+ total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);
+
+ kfd_queue_buffer_svm_put(pdd, properties->ctx_save_restore_area_address, total_cwsr_size);
+diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+index ad9125f3655fae..83f13c65c0a571 100644
+--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
++++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+@@ -5826,8 +5826,8 @@ static void fill_dc_dirty_rects(struct drm_plane *plane,
+ {
+ struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
+ struct rect *dirty_rects = flip_addrs->dirty_rects;
+- u32 num_clips;
+- struct drm_mode_rect *clips;
++ u32 num_clips = 0;
++ struct drm_mode_rect *clips = NULL;
+ bool bb_changed;
+ bool fb_changed;
+ u32 i = 0;
+@@ -5843,8 +5843,10 @@ static void fill_dc_dirty_rects(struct drm_plane *plane,
+ if (new_plane_state->rotation != DRM_MODE_ROTATE_0)
+ goto ffu;
+
+- num_clips = drm_plane_get_damage_clips_count(new_plane_state);
+- clips = drm_plane_get_damage_clips(new_plane_state);
++ if (!new_plane_state->ignore_damage_clips) {
++ num_clips = drm_plane_get_damage_clips_count(new_plane_state);
++ clips = drm_plane_get_damage_clips(new_plane_state);
++ }
+
+ if (num_clips && (!amdgpu_damage_clips || (amdgpu_damage_clips < 0 &&
+ is_psr_su)))
+@@ -10834,6 +10836,7 @@ skip_modeset:
+ /* Release extra reference */
+ if (new_stream)
+ dc_stream_release(new_stream);
++ new_stream = NULL;
+
+ /*
+ * We want to do dc stream updates that do not require a
+diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
+index 084994c650c4c9..285689d7ecc4ea 100644
+--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
++++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
+@@ -276,13 +276,20 @@ static void dcn32_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr,
+ struct dtbclk_dto_params dto_params = {0};
+
+ /* use mask to program DTO once per tg */
+- if (pipe_ctx->stream_res.tg &&
++ if (pipe_ctx->stream && pipe_ctx->stream_res.tg &&
+ !(tg_mask & (1 << pipe_ctx->stream_res.tg->inst))) {
+ tg_mask |= (1 << pipe_ctx->stream_res.tg->inst);
+
+ dto_params.otg_inst = pipe_ctx->stream_res.tg->inst;
+ dto_params.ref_dtbclk_khz = ref_dtbclk_khz;
+
++ if (dccg->ctx->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx))
++ dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
++
++ if (dc_is_hdmi_signal(pipe_ctx->stream->signal) ||
++ dc_is_dvi_signal(pipe_ctx->stream->signal))
++ dto_params.is_hdmi = true;
++
+ dccg->funcs->set_dtbclk_dto(clk_mgr->dccg, &dto_params);
+ //dccg->funcs->set_audio_dtbclk_dto(clk_mgr->dccg, &dto_params);
+ }
+diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
+index 33ee7e81b1b939..05e4dbc7fb8398 100644
+--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
++++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
+@@ -247,13 +247,20 @@ static void dcn35_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr,
+ struct dtbclk_dto_params dto_params = {0};
+
+ /* use mask to program DTO once per tg */
+- if (pipe_ctx->stream_res.tg &&
++ if (pipe_ctx->stream && pipe_ctx->stream_res.tg &&
+ !(tg_mask & (1 << pipe_ctx->stream_res.tg->inst))) {
+ tg_mask |= (1 << pipe_ctx->stream_res.tg->inst);
+
+ dto_params.otg_inst = pipe_ctx->stream_res.tg->inst;
+ dto_params.ref_dtbclk_khz = ref_dtbclk_khz;
+
++ if (dccg->ctx->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx))
++ dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
++
++ if (dc_is_hdmi_signal(pipe_ctx->stream->signal) ||
++ dc_is_dvi_signal(pipe_ctx->stream->signal))
++ dto_params.is_hdmi = true;
++
+ dccg->funcs->set_dtbclk_dto(clk_mgr->dccg, &dto_params);
+ //dccg->funcs->set_audio_dtbclk_dto(clk_mgr->dccg, &dto_params);
+ }
+diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+index 4e1035c28e18a0..c1295a1425924e 100644
+--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
++++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+@@ -961,8 +961,11 @@ static bool detect_link_and_local_sink(struct dc_link *link,
+ link->link_enc->features.flags.bits.DP_IS_USB_C == 1) {
+
+ /* if alt mode times out, return false */
+- if (!wait_for_entering_dp_alt_mode(link))
++ if (!wait_for_entering_dp_alt_mode(link)) {
++ if (prev_sink)
++ dc_sink_release(prev_sink);
+ return false;
++ }
+ }
+
+ if (!detect_dp(link, &sink_caps, reason)) {
+diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
+index 53a5f4cb648cbd..e0ef369d4ab744 100644
+--- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
++++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
+@@ -939,6 +939,11 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
+ for (i = 0; i < pool->stream_enc_count; i++) {
+ if (!res_ctx->is_stream_enc_acquired[i] &&
+ pool->stream_enc[i]) {
++ /* DP/MST needs a digital encoder; skip analog/no-DP encoders */
++ if (dc_is_dp_signal(stream->signal) &&
++ (!pool->stream_enc[i]->funcs ||
++ !pool->stream_enc[i]->funcs->dp_set_stream_attribute))
++ continue;
+ /* Store first available for MST second display
+ * in daisy chain use case
+ */
+@@ -962,7 +967,7 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
+ * required for non DP connectors.
+ */
+
+- if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT)
++ if (j >= 0 && dc_is_dp_signal(stream->signal))
+ return pool->stream_enc[j];
+
+ return NULL;
+diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+index ae858a43c35f4a..b3d55cac35694b 100644
+--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
++++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+@@ -447,8 +447,6 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
+ *
+ * @stream: contains data we may need to construct VSIF (i.e. timing_3d_format, etc.)
+ * @info_packet: output structure where to store VSIF
+- * @ALLMEnabled: indicates whether ALLM HF-VSIF should be generated
+- * @ALLMValue: ALLM bit value to advertise in HF-VSIF
+ */
+ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream,
+ struct dc_info_packet *info_packet)
+diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+index 9ab0a7fc3cd9be..2252c054c7ddd3 100644
+--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
++++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+@@ -40,6 +40,8 @@
+
+ #define DEVICE_ATTR_IS(_name) (attr_id == device_attr_id__##_name)
+
++#define power_2_mwatt(power) (((power) >> 8) * 1000 + ((power) & 0xff))
++
+ struct od_attribute {
+ struct kobj_attribute attribute;
+ struct list_head entry;
+@@ -2507,6 +2509,11 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
+ gc_ver != IP_VERSION(9, 4, 3)) ||
+ gc_ver < IP_VERSION(9, 0, 0))
+ *states = ATTR_STATE_UNSUPPORTED;
++
++ if (adev->scpm_enabled) {
++ dev_attr->attr.mode &= ~S_IWUGO;
++ dev_attr->store = NULL;
++ }
+ } else if (DEVICE_ATTR_IS(gpu_metrics)) {
+ if (gc_ver < IP_VERSION(9, 1, 0))
+ *states = ATTR_STATE_UNSUPPORTED;
+@@ -3200,7 +3207,6 @@ static int amdgpu_hwmon_get_power(struct device *dev,
+ enum amd_pp_sensors sensor)
+ {
+ struct amdgpu_device *adev = dev_get_drvdata(dev);
+- unsigned int uw;
+ u32 query = 0;
+ int r;
+
+@@ -3209,9 +3215,7 @@ static int amdgpu_hwmon_get_power(struct device *dev,
+ return r;
+
+ /* convert to microwatts */
+- uw = (query >> 8) * 1000000 + (query & 0xff) * 1000;
+-
+- return uw;
++ return power_2_mwatt(query) * 1000;
+ }
+
+ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev,
+@@ -4607,7 +4611,7 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a
+ {
+ uint32_t mp1_ver = amdgpu_ip_version(adev, MP1_HWIP, 0);
+ uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+- uint32_t value;
++ uint32_t value, mwatt, centiwatt;
+ uint64_t value64 = 0;
+ uint32_t query = 0;
+ int size;
+@@ -4632,17 +4636,21 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a
+ seq_printf(m, "\t%u mV (VDDNB)\n", value);
+ size = sizeof(uint32_t);
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_AVG_POWER, (void *)&query, &size)) {
++ mwatt = power_2_mwatt(query);
++ centiwatt = DIV_ROUND_CLOSEST(mwatt, 10);
+ if (adev->flags & AMD_IS_APU)
+- seq_printf(m, "\t%u.%02u W (average SoC including CPU)\n", query >> 8, query & 0xff);
++ seq_printf(m, "\t%u.%02u W (average SoC including CPU)\n", centiwatt / 100, centiwatt % 100);
+ else
+- seq_printf(m, "\t%u.%02u W (average SoC)\n", query >> 8, query & 0xff);
++ seq_printf(m, "\t%u.%02u W (average SoC)\n", centiwatt / 100, centiwatt % 100);
+ }
+ size = sizeof(uint32_t);
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_INPUT_POWER, (void *)&query, &size)) {
++ mwatt = power_2_mwatt(query);
++ centiwatt = DIV_ROUND_CLOSEST(mwatt, 10);
+ if (adev->flags & AMD_IS_APU)
+- seq_printf(m, "\t%u.%02u W (current SoC including CPU)\n", query >> 8, query & 0xff);
++ seq_printf(m, "\t%u.%02u W (current SoC including CPU)\n", centiwatt / 100, centiwatt % 100);
+ else
+- seq_printf(m, "\t%u.%02u W (current SoC)\n", query >> 8, query & 0xff);
++ seq_printf(m, "\t%u.%02u W (current SoC)\n", centiwatt / 100, centiwatt % 100);
+ }
+ size = sizeof(value);
+ seq_printf(m, "\n");
+diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c
+index 1d6e30269d5679..4d553be56396f3 100644
+--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c
++++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c
+@@ -106,11 +106,8 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr)
+ hwmgr->od_enabled = false;
+ switch (hwmgr->chip_id) {
+ case CHIP_BONAIRE:
+- /* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM
+- * R7 260X cards with old MC ucode: MCLK DPM is unstable
+- */
+- if (adev->pdev->subsystem_vendor == 0x106B ||
+- adev->pdev->device == 0x6658) {
++ /* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM */
++ if (adev->pdev->subsystem_vendor == 0x106B) {
+ dev_info(adev->dev, "disabling MCLK DPM on quirky ASIC");
+ adev->pm.pp_feature &= ~PP_MCLK_DPM_MASK;
+ hwmgr->feature_mask &= ~PP_MCLK_DPM_MASK;
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+index bef3bd11fde0cc..7ecc7537ae2f3e 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+@@ -2396,11 +2396,14 @@ static int smu_v13_0_0_get_power_limit(struct smu_context *smu,
+ uint32_t pp_limit = smu->adev->pm.ac_power ?
+ skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] :
+ skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0];
+- uint32_t power_limit = 0, od_percent_upper = 0, od_percent_lower = 0;
++ uint32_t msg_limit = skutable->MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC];
++ uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit);
++ uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit);
++ uint32_t od_percent_upper = 0, od_percent_lower = 0;
+ int ret;
+
+ if (current_power_limit) {
+- ret = smu_v13_0_get_current_power_limit(smu, &power_limit);
++ ret = smu_v13_0_get_current_power_limit(smu, current_power_limit);
+ if (ret)
+ *current_power_limit = pp_limit;
+ }
+@@ -2423,12 +2426,12 @@ static int smu_v13_0_0_get_power_limit(struct smu_context *smu,
+ od_percent_upper, od_percent_lower, pp_limit);
+
+ if (max_power_limit) {
+- *max_power_limit = pp_limit * (100 + od_percent_upper);
++ *max_power_limit = max_limit * (100 + od_percent_upper);
+ *max_power_limit /= 100;
+ }
+
+ if (min_power_limit) {
+- *min_power_limit = pp_limit * (100 - od_percent_lower);
++ *min_power_limit = min_limit * (100 - od_percent_lower);
+ *min_power_limit /= 100;
+ }
+
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+index 8caf4482929084..46368fc966435e 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+@@ -2357,15 +2357,16 @@ static int smu_v13_0_7_get_power_limit(struct smu_context *smu,
+ uint32_t pp_limit = smu->adev->pm.ac_power ?
+ skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] :
+ skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0];
+- uint32_t power_limit = 0, od_percent_upper = 0, od_percent_lower = 0;
++ uint32_t msg_limit = skutable->MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC];
++ uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit);
++ uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit);
++ uint32_t od_percent_upper = 0, od_percent_lower = 0;
+ int ret;
+
+ if (current_power_limit) {
+- ret = smu_v13_0_get_current_power_limit(smu, &power_limit);
++ ret = smu_v13_0_get_current_power_limit(smu, current_power_limit);
+ if (ret)
+- power_limit = pp_limit;
+-
+- *current_power_limit = power_limit;
++ *current_power_limit = pp_limit;
+ }
+
+ if (default_power_limit)
+@@ -2386,12 +2387,12 @@ static int smu_v13_0_7_get_power_limit(struct smu_context *smu,
+ od_percent_upper, od_percent_lower, pp_limit);
+
+ if (max_power_limit) {
+- *max_power_limit = pp_limit * (100 + od_percent_upper);
++ *max_power_limit = max_limit * (100 + od_percent_upper);
+ *max_power_limit /= 100;
+ }
+
+ if (min_power_limit) {
+- *min_power_limit = pp_limit * (100 - od_percent_lower);
++ *min_power_limit = min_limit * (100 - od_percent_lower);
+ *min_power_limit /= 100;
+ }
+
+diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
+index 29dd41fd853915..b2c32b2d2e510f 100644
+--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
++++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
+@@ -1672,19 +1672,23 @@ static int smu_v14_0_2_get_power_limit(struct smu_context *smu,
+ table_context->power_play_table;
+ PPTable_t *pptable = table_context->driver_pptable;
+ CustomSkuTable_t *skutable = &pptable->CustomSkuTable;
+- int16_t od_percent_upper = 0, od_percent_lower = 0;
++ uint32_t pp_limit = smu->adev->pm.ac_power ?
++ skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] :
++ skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0];
+ uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC];
+- uint32_t power_limit;
++ uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit);
++ uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit);
++ int16_t od_percent_upper = 0, od_percent_lower = 0;
++ int ret;
+
+- if (smu_v14_0_get_current_power_limit(smu, &power_limit))
+- power_limit = smu->adev->pm.ac_power ?
+- skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] :
+- skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0];
++ if (current_power_limit) {
++ ret = smu_v14_0_get_current_power_limit(smu, current_power_limit);
++ if (ret)
++ *current_power_limit = pp_limit;
++ }
+
+- if (current_power_limit)
+- *current_power_limit = power_limit;
+ if (default_power_limit)
+- *default_power_limit = power_limit;
++ *default_power_limit = pp_limit;
+
+ if (powerplay_table) {
+ if (smu->od_enabled &&
+@@ -1698,15 +1702,15 @@ static int smu_v14_0_2_get_power_limit(struct smu_context *smu,
+ }
+
+ dev_dbg(smu->adev->dev, "od percent upper:%d, od percent lower:%d (default power: %d)\n",
+- od_percent_upper, od_percent_lower, power_limit);
++ od_percent_upper, od_percent_lower, pp_limit);
+
+ if (max_power_limit) {
+- *max_power_limit = msg_limit * (100 + od_percent_upper);
++ *max_power_limit = max_limit * (100 + od_percent_upper);
+ *max_power_limit /= 100;
+ }
+
+ if (min_power_limit) {
+- *min_power_limit = power_limit * (100 + od_percent_lower);
++ *min_power_limit = min_limit * (100 + od_percent_lower);
+ *min_power_limit /= 100;
+ }
+
+diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
+index ddfbb2009c8d38..790a559e9549ba 100644
+--- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
++++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
+@@ -1157,7 +1157,7 @@ static const struct mipi_dsi_host_ops cdns_dsi_ops = {
+ .transfer = cdns_dsi_transfer,
+ };
+
+-static int __maybe_unused cdns_dsi_resume(struct device *dev)
++static int cdns_dsi_resume(struct device *dev)
+ {
+ struct cdns_dsi *dsi = dev_get_drvdata(dev);
+
+@@ -1168,7 +1168,7 @@ static int __maybe_unused cdns_dsi_resume(struct device *dev)
+ return 0;
+ }
+
+-static int __maybe_unused cdns_dsi_suspend(struct device *dev)
++static int cdns_dsi_suspend(struct device *dev)
+ {
+ struct cdns_dsi *dsi = dev_get_drvdata(dev);
+
+@@ -1178,8 +1178,9 @@ static int __maybe_unused cdns_dsi_suspend(struct device *dev)
+ return 0;
+ }
+
+-static UNIVERSAL_DEV_PM_OPS(cdns_dsi_pm_ops, cdns_dsi_suspend, cdns_dsi_resume,
+- NULL);
++static const struct dev_pm_ops cdns_dsi_pm_ops = {
++ RUNTIME_PM_OPS(cdns_dsi_suspend, cdns_dsi_resume, NULL)
++};
+
+ static int cdns_dsi_drm_probe(struct platform_device *pdev)
+ {
+@@ -1326,7 +1327,7 @@ static struct platform_driver cdns_dsi_platform_driver = {
+ .driver = {
+ .name = "cdns-dsi",
+ .of_match_table = cdns_dsi_of_match,
+- .pm = &cdns_dsi_pm_ops,
++ .pm = pm_ptr(&cdns_dsi_pm_ops),
+ },
+ };
+ module_platform_driver(cdns_dsi_platform_driver);
+diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
+index 997f2489f00f3b..30cbc5f28838d7 100644
+--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
++++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
+@@ -780,6 +780,12 @@ static bool drm_dp_sideband_append_payload(struct drm_dp_sideband_msg_rx *msg,
+ {
+ u8 crc4;
+
++ /* curchunk_len must be >= 1 (min 1 CRC byte) and fit in chunk[] */
++ if (!msg->curchunk_len ||
++ msg->curchunk_len > ARRAY_SIZE(msg->chunk) ||
++ msg->curchunk_idx + replybuflen > ARRAY_SIZE(msg->chunk))
++ return false;
++
+ memcpy(&msg->chunk[msg->curchunk_idx], replybuf, replybuflen);
+ msg->curchunk_idx += replybuflen;
+
+@@ -790,6 +796,9 @@ static bool drm_dp_sideband_append_payload(struct drm_dp_sideband_msg_rx *msg,
+ print_hex_dump(KERN_DEBUG, "wrong crc",
+ DUMP_PREFIX_NONE, 16, 1,
+ msg->chunk, msg->curchunk_len, false);
++ /* Guard against accumulated msg[] overflow */
++ if (msg->curlen + msg->curchunk_len - 1 > ARRAY_SIZE(msg->msg))
++ return false;
+ /* copy chunk into bigger msg */
+ memcpy(&msg->msg[msg->curlen], msg->chunk, msg->curchunk_len - 1);
+ msg->curlen += msg->curchunk_len - 1;
+@@ -862,7 +871,7 @@ static bool drm_dp_sideband_parse_remote_dpcd_read(struct drm_dp_sideband_msg_rx
+ goto fail_len;
+ repmsg->u.remote_dpcd_read_ack.num_bytes = raw->msg[idx];
+ idx++;
+- if (idx > raw->curlen)
++ if (idx + repmsg->u.remote_dpcd_read_ack.num_bytes > raw->curlen)
+ goto fail_len;
+
+ memcpy(repmsg->u.remote_dpcd_read_ack.bytes, &raw->msg[idx], repmsg->u.remote_dpcd_read_ack.num_bytes);
+@@ -898,7 +907,9 @@ static bool drm_dp_sideband_parse_remote_i2c_read_ack(struct drm_dp_sideband_msg
+ goto fail_len;
+ repmsg->u.remote_i2c_read_ack.num_bytes = raw->msg[idx];
+ idx++;
+- /* TODO check */
++ if (idx + repmsg->u.remote_i2c_read_ack.num_bytes > raw->curlen)
++ goto fail_len;
++
+ memcpy(repmsg->u.remote_i2c_read_ack.bytes, &raw->msg[idx], repmsg->u.remote_i2c_read_ack.num_bytes);
+ return true;
+ fail_len:
+@@ -914,16 +925,13 @@ static bool drm_dp_sideband_parse_enum_path_resources_ack(struct drm_dp_sideband
+ repmsg->u.path_resources.port_number = (raw->msg[idx] >> 4) & 0xf;
+ repmsg->u.path_resources.fec_capable = raw->msg[idx] & 0x1;
+ idx++;
+- if (idx > raw->curlen)
++ if (idx + 2 > raw->curlen)
+ goto fail_len;
+ repmsg->u.path_resources.full_payload_bw_number = (raw->msg[idx] << 8) | (raw->msg[idx+1]);
+ idx += 2;
+- if (idx > raw->curlen)
++ if (idx + 2 > raw->curlen)
+ goto fail_len;
+ repmsg->u.path_resources.avail_payload_bw_number = (raw->msg[idx] << 8) | (raw->msg[idx+1]);
+- idx += 2;
+- if (idx > raw->curlen)
+- goto fail_len;
+ return true;
+ fail_len:
+ DRM_DEBUG_KMS("enum resource parse length fail %d %d\n", idx, raw->curlen);
+@@ -941,12 +949,9 @@ static bool drm_dp_sideband_parse_allocate_payload_ack(struct drm_dp_sideband_ms
+ goto fail_len;
+ repmsg->u.allocate_payload.vcpi = raw->msg[idx];
+ idx++;
+- if (idx > raw->curlen)
++ if (idx + 2 > raw->curlen)
+ goto fail_len;
+ repmsg->u.allocate_payload.allocated_pbn = (raw->msg[idx] << 8) | (raw->msg[idx+1]);
+- idx += 2;
+- if (idx > raw->curlen)
+- goto fail_len;
+ return true;
+ fail_len:
+ DRM_DEBUG_KMS("allocate payload parse length fail %d %d\n", idx, raw->curlen);
+@@ -960,12 +965,9 @@ static bool drm_dp_sideband_parse_query_payload_ack(struct drm_dp_sideband_msg_r
+
+ repmsg->u.query_payload.port_number = (raw->msg[idx] >> 4) & 0xf;
+ idx++;
+- if (idx > raw->curlen)
++ if (idx + 2 > raw->curlen)
+ goto fail_len;
+ repmsg->u.query_payload.allocated_pbn = (raw->msg[idx] << 8) | (raw->msg[idx + 1]);
+- idx += 2;
+- if (idx > raw->curlen)
+- goto fail_len;
+ return true;
+ fail_len:
+ DRM_DEBUG_KMS("query payload parse length fail %d %d\n", idx, raw->curlen);
+@@ -3741,8 +3743,10 @@ void drm_dp_mst_topology_queue_probe(struct drm_dp_mst_topology_mgr *mgr)
+ {
+ mutex_lock(&mgr->lock);
+
+- if (drm_WARN_ON(mgr->dev, !mgr->mst_state || !mgr->mst_primary))
++ if (!mgr->mst_state || !mgr->mst_primary) {
++ drm_dbg_kms(mgr->dev, "queue_probe skipped: topology torn down\n");
+ goto out_unlock;
++ }
+
+ drm_dp_mst_topology_mgr_invalidate_mstb(mgr->mst_primary);
+ drm_dp_mst_queue_probe_work(mgr);
+diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c
+index adfcdd5645d9a7..841f3de5f307a0 100644
+--- a/drivers/gpu/drm/drm_buddy.c
++++ b/drivers/gpu/drm/drm_buddy.c
+@@ -3,1252 +3,16 @@
+ * Copyright © 2021 Intel Corporation
+ */
+
++#include <kunit/test-bug.h>
++
++#include <linux/export.h>
+ #include <linux/kmemleak.h>
+ #include <linux/module.h>
+ #include <linux/sizes.h>
+
++#include <linux/gpu_buddy.h>
+ #include <drm/drm_buddy.h>
+-
+-enum drm_buddy_free_tree {
+- DRM_BUDDY_CLEAR_TREE = 0,
+- DRM_BUDDY_DIRTY_TREE,
+- DRM_BUDDY_MAX_FREE_TREES,
+-};
+-
+-static struct kmem_cache *slab_blocks;
+-
+-#define for_each_free_tree(tree) \
+- for ((tree) = 0; (tree) < DRM_BUDDY_MAX_FREE_TREES; (tree)++)
+-
+-static struct drm_buddy_block *drm_block_alloc(struct drm_buddy *mm,
+- struct drm_buddy_block *parent,
+- unsigned int order,
+- u64 offset)
+-{
+- struct drm_buddy_block *block;
+-
+- BUG_ON(order > DRM_BUDDY_MAX_ORDER);
+-
+- block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL);
+- if (!block)
+- return NULL;
+-
+- block->header = offset;
+- block->header |= order;
+- block->parent = parent;
+-
+- RB_CLEAR_NODE(&block->rb);
+-
+- BUG_ON(block->header & DRM_BUDDY_HEADER_UNUSED);
+- return block;
+-}
+-
+-static void drm_block_free(struct drm_buddy *mm,
+- struct drm_buddy_block *block)
+-{
+- kmem_cache_free(slab_blocks, block);
+-}
+-
+-static enum drm_buddy_free_tree
+-get_block_tree(struct drm_buddy_block *block)
+-{
+- return drm_buddy_block_is_clear(block) ?
+- DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE;
+-}
+-
+-static struct drm_buddy_block *
+-rbtree_get_free_block(const struct rb_node *node)
+-{
+- return node ? rb_entry(node, struct drm_buddy_block, rb) : NULL;
+-}
+-
+-static struct drm_buddy_block *
+-rbtree_last_free_block(struct rb_root *root)
+-{
+- return rbtree_get_free_block(rb_last(root));
+-}
+-
+-static bool rbtree_is_empty(struct rb_root *root)
+-{
+- return RB_EMPTY_ROOT(root);
+-}
+-
+-static bool drm_buddy_block_offset_less(const struct drm_buddy_block *block,
+- const struct drm_buddy_block *node)
+-{
+- return drm_buddy_block_offset(block) < drm_buddy_block_offset(node);
+-}
+-
+-static bool rbtree_block_offset_less(struct rb_node *block,
+- const struct rb_node *node)
+-{
+- return drm_buddy_block_offset_less(rbtree_get_free_block(block),
+- rbtree_get_free_block(node));
+-}
+-
+-static void rbtree_insert(struct drm_buddy *mm,
+- struct drm_buddy_block *block,
+- enum drm_buddy_free_tree tree)
+-{
+- rb_add(&block->rb,
+- &mm->free_trees[tree][drm_buddy_block_order(block)],
+- rbtree_block_offset_less);
+-}
+-
+-static void rbtree_remove(struct drm_buddy *mm,
+- struct drm_buddy_block *block)
+-{
+- unsigned int order = drm_buddy_block_order(block);
+- enum drm_buddy_free_tree tree;
+- struct rb_root *root;
+-
+- tree = get_block_tree(block);
+- root = &mm->free_trees[tree][order];
+-
+- rb_erase(&block->rb, root);
+- RB_CLEAR_NODE(&block->rb);
+-}
+-
+-static void clear_reset(struct drm_buddy_block *block)
+-{
+- block->header &= ~DRM_BUDDY_HEADER_CLEAR;
+-}
+-
+-static void mark_cleared(struct drm_buddy_block *block)
+-{
+- block->header |= DRM_BUDDY_HEADER_CLEAR;
+-}
+-
+-static void mark_allocated(struct drm_buddy *mm,
+- struct drm_buddy_block *block)
+-{
+- block->header &= ~DRM_BUDDY_HEADER_STATE;
+- block->header |= DRM_BUDDY_ALLOCATED;
+-
+- rbtree_remove(mm, block);
+-}
+-
+-static void mark_free(struct drm_buddy *mm,
+- struct drm_buddy_block *block)
+-{
+- enum drm_buddy_free_tree tree;
+-
+- block->header &= ~DRM_BUDDY_HEADER_STATE;
+- block->header |= DRM_BUDDY_FREE;
+-
+- tree = get_block_tree(block);
+- rbtree_insert(mm, block, tree);
+-}
+-
+-static void mark_split(struct drm_buddy *mm,
+- struct drm_buddy_block *block)
+-{
+- block->header &= ~DRM_BUDDY_HEADER_STATE;
+- block->header |= DRM_BUDDY_SPLIT;
+-
+- rbtree_remove(mm, block);
+-}
+-
+-static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
+-{
+- return s1 <= e2 && e1 >= s2;
+-}
+-
+-static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
+-{
+- return s1 <= s2 && e1 >= e2;
+-}
+-
+-static struct drm_buddy_block *
+-__get_buddy(struct drm_buddy_block *block)
+-{
+- struct drm_buddy_block *parent;
+-
+- parent = block->parent;
+- if (!parent)
+- return NULL;
+-
+- if (parent->left == block)
+- return parent->right;
+-
+- return parent->left;
+-}
+-
+-static unsigned int __drm_buddy_free(struct drm_buddy *mm,
+- struct drm_buddy_block *block,
+- bool force_merge)
+-{
+- struct drm_buddy_block *parent;
+- unsigned int order;
+-
+- while ((parent = block->parent)) {
+- struct drm_buddy_block *buddy;
+-
+- buddy = __get_buddy(block);
+-
+- if (!drm_buddy_block_is_free(buddy))
+- break;
+-
+- if (!force_merge) {
+- /*
+- * Check the block and its buddy clear state and exit
+- * the loop if they both have the dissimilar state.
+- */
+- if (drm_buddy_block_is_clear(block) !=
+- drm_buddy_block_is_clear(buddy))
+- break;
+-
+- if (drm_buddy_block_is_clear(block))
+- mark_cleared(parent);
+- }
+-
+- rbtree_remove(mm, buddy);
+- if (force_merge && drm_buddy_block_is_clear(buddy))
+- mm->clear_avail -= drm_buddy_block_size(mm, buddy);
+-
+- drm_block_free(mm, block);
+- drm_block_free(mm, buddy);
+-
+- block = parent;
+- }
+-
+- order = drm_buddy_block_order(block);
+- mark_free(mm, block);
+-
+- return order;
+-}
+-
+-static int __force_merge(struct drm_buddy *mm,
+- u64 start,
+- u64 end,
+- unsigned int min_order)
+-{
+- unsigned int tree, order;
+- int i;
+-
+- if (!min_order)
+- return -ENOMEM;
+-
+- if (min_order > mm->max_order)
+- return -EINVAL;
+-
+- for_each_free_tree(tree) {
+- for (i = min_order - 1; i >= 0; i--) {
+- struct rb_node *iter = rb_last(&mm->free_trees[tree][i]);
+-
+- while (iter) {
+- struct drm_buddy_block *block, *buddy;
+- u64 block_start, block_end;
+-
+- block = rbtree_get_free_block(iter);
+- iter = rb_prev(iter);
+-
+- if (!block || !block->parent)
+- continue;
+-
+- block_start = drm_buddy_block_offset(block);
+- block_end = block_start + drm_buddy_block_size(mm, block) - 1;
+-
+- if (!contains(start, end, block_start, block_end))
+- continue;
+-
+- buddy = __get_buddy(block);
+- if (!drm_buddy_block_is_free(buddy))
+- continue;
+-
+- WARN_ON(drm_buddy_block_is_clear(block) ==
+- drm_buddy_block_is_clear(buddy));
+-
+- /*
+- * Advance to the next node when the current node is the buddy,
+- * as freeing the block will also remove its buddy from the tree.
+- */
+- if (iter == &buddy->rb)
+- iter = rb_prev(iter);
+-
+- rbtree_remove(mm, block);
+- if (drm_buddy_block_is_clear(block))
+- mm->clear_avail -= drm_buddy_block_size(mm, block);
+-
+- order = __drm_buddy_free(mm, block, true);
+- if (order >= min_order)
+- return 0;
+- }
+- }
+- }
+-
+- return -ENOMEM;
+-}
+-
+-/**
+- * drm_buddy_init - init memory manager
+- *
+- * @mm: DRM buddy manager to initialize
+- * @size: size in bytes to manage
+- * @chunk_size: minimum page size in bytes for our allocations
+- *
+- * Initializes the memory manager and its resources.
+- *
+- * Returns:
+- * 0 on success, error code on failure.
+- */
+-int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size)
+-{
+- unsigned int i, j, root_count = 0;
+- u64 offset = 0;
+-
+- if (size < chunk_size)
+- return -EINVAL;
+-
+- if (chunk_size < SZ_4K)
+- return -EINVAL;
+-
+- if (!is_power_of_2(chunk_size))
+- return -EINVAL;
+-
+- size = round_down(size, chunk_size);
+-
+- mm->size = size;
+- mm->avail = size;
+- mm->clear_avail = 0;
+- mm->chunk_size = chunk_size;
+- mm->max_order = ilog2(size) - ilog2(chunk_size);
+-
+- BUG_ON(mm->max_order > DRM_BUDDY_MAX_ORDER);
+-
+- mm->free_trees = kmalloc_array(DRM_BUDDY_MAX_FREE_TREES,
+- sizeof(*mm->free_trees),
+- GFP_KERNEL);
+- if (!mm->free_trees)
+- return -ENOMEM;
+-
+- for_each_free_tree(i) {
+- mm->free_trees[i] = kmalloc_array(mm->max_order + 1,
+- sizeof(struct rb_root),
+- GFP_KERNEL);
+- if (!mm->free_trees[i])
+- goto out_free_tree;
+-
+- for (j = 0; j <= mm->max_order; ++j)
+- mm->free_trees[i][j] = RB_ROOT;
+- }
+-
+- mm->n_roots = hweight64(size);
+-
+- mm->roots = kmalloc_array(mm->n_roots,
+- sizeof(struct drm_buddy_block *),
+- GFP_KERNEL);
+- if (!mm->roots)
+- goto out_free_tree;
+-
+- /*
+- * Split into power-of-two blocks, in case we are given a size that is
+- * not itself a power-of-two.
+- */
+- do {
+- struct drm_buddy_block *root;
+- unsigned int order;
+- u64 root_size;
+-
+- order = ilog2(size) - ilog2(chunk_size);
+- root_size = chunk_size << order;
+-
+- root = drm_block_alloc(mm, NULL, order, offset);
+- if (!root)
+- goto out_free_roots;
+-
+- mark_free(mm, root);
+-
+- BUG_ON(root_count > mm->max_order);
+- BUG_ON(drm_buddy_block_size(mm, root) < chunk_size);
+-
+- mm->roots[root_count] = root;
+-
+- offset += root_size;
+- size -= root_size;
+- root_count++;
+- } while (size);
+-
+- return 0;
+-
+-out_free_roots:
+- while (root_count--)
+- drm_block_free(mm, mm->roots[root_count]);
+- kfree(mm->roots);
+-out_free_tree:
+- while (i--)
+- kfree(mm->free_trees[i]);
+- kfree(mm->free_trees);
+- return -ENOMEM;
+-}
+-EXPORT_SYMBOL(drm_buddy_init);
+-
+-/**
+- * drm_buddy_fini - tear down the memory manager
+- *
+- * @mm: DRM buddy manager to free
+- *
+- * Cleanup memory manager resources and the freetree
+- */
+-void drm_buddy_fini(struct drm_buddy *mm)
+-{
+- u64 root_size, size, start;
+- unsigned int order;
+- int i;
+-
+- size = mm->size;
+-
+- for (i = 0; i < mm->n_roots; ++i) {
+- order = ilog2(size) - ilog2(mm->chunk_size);
+- start = drm_buddy_block_offset(mm->roots[i]);
+- __force_merge(mm, start, start + size, order);
+-
+- WARN_ON(!drm_buddy_block_is_free(mm->roots[i]));
+- drm_block_free(mm, mm->roots[i]);
+-
+- root_size = mm->chunk_size << order;
+- size -= root_size;
+- }
+-
+- WARN_ON(mm->avail != mm->size);
+-
+- for_each_free_tree(i)
+- kfree(mm->free_trees[i]);
+- kfree(mm->free_trees);
+- kfree(mm->roots);
+-}
+-EXPORT_SYMBOL(drm_buddy_fini);
+-
+-static int split_block(struct drm_buddy *mm,
+- struct drm_buddy_block *block)
+-{
+- unsigned int block_order = drm_buddy_block_order(block) - 1;
+- u64 offset = drm_buddy_block_offset(block);
+-
+- BUG_ON(!drm_buddy_block_is_free(block));
+- BUG_ON(!drm_buddy_block_order(block));
+-
+- block->left = drm_block_alloc(mm, block, block_order, offset);
+- if (!block->left)
+- return -ENOMEM;
+-
+- block->right = drm_block_alloc(mm, block, block_order,
+- offset + (mm->chunk_size << block_order));
+- if (!block->right) {
+- drm_block_free(mm, block->left);
+- return -ENOMEM;
+- }
+-
+- mark_split(mm, block);
+-
+- if (drm_buddy_block_is_clear(block)) {
+- mark_cleared(block->left);
+- mark_cleared(block->right);
+- clear_reset(block);
+- }
+-
+- mark_free(mm, block->left);
+- mark_free(mm, block->right);
+-
+- return 0;
+-}
+-
+-/**
+- * drm_get_buddy - get buddy address
+- *
+- * @block: DRM buddy block
+- *
+- * Returns the corresponding buddy block for @block, or NULL
+- * if this is a root block and can't be merged further.
+- * Requires some kind of locking to protect against
+- * any concurrent allocate and free operations.
+- */
+-struct drm_buddy_block *
+-drm_get_buddy(struct drm_buddy_block *block)
+-{
+- return __get_buddy(block);
+-}
+-EXPORT_SYMBOL(drm_get_buddy);
+-
+-/**
+- * drm_buddy_reset_clear - reset blocks clear state
+- *
+- * @mm: DRM buddy manager
+- * @is_clear: blocks clear state
+- *
+- * Reset the clear state based on @is_clear value for each block
+- * in the freetree.
+- */
+-void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear)
+-{
+- enum drm_buddy_free_tree src_tree, dst_tree;
+- u64 root_size, size, start;
+- unsigned int order;
+- int i;
+-
+- size = mm->size;
+- for (i = 0; i < mm->n_roots; ++i) {
+- order = ilog2(size) - ilog2(mm->chunk_size);
+- start = drm_buddy_block_offset(mm->roots[i]);
+- __force_merge(mm, start, start + size, order);
+-
+- root_size = mm->chunk_size << order;
+- size -= root_size;
+- }
+-
+- src_tree = is_clear ? DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE;
+- dst_tree = is_clear ? DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE;
+-
+- for (i = 0; i <= mm->max_order; ++i) {
+- struct rb_root *root = &mm->free_trees[src_tree][i];
+- struct drm_buddy_block *block, *tmp;
+-
+- rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
+- rbtree_remove(mm, block);
+- if (is_clear) {
+- mark_cleared(block);
+- mm->clear_avail += drm_buddy_block_size(mm, block);
+- } else {
+- clear_reset(block);
+- mm->clear_avail -= drm_buddy_block_size(mm, block);
+- }
+-
+- rbtree_insert(mm, block, dst_tree);
+- }
+- }
+-}
+-EXPORT_SYMBOL(drm_buddy_reset_clear);
+-
+-/**
+- * drm_buddy_free_block - free a block
+- *
+- * @mm: DRM buddy manager
+- * @block: block to be freed
+- */
+-void drm_buddy_free_block(struct drm_buddy *mm,
+- struct drm_buddy_block *block)
+-{
+- BUG_ON(!drm_buddy_block_is_allocated(block));
+- mm->avail += drm_buddy_block_size(mm, block);
+- if (drm_buddy_block_is_clear(block))
+- mm->clear_avail += drm_buddy_block_size(mm, block);
+-
+- __drm_buddy_free(mm, block, false);
+-}
+-EXPORT_SYMBOL(drm_buddy_free_block);
+-
+-static void __drm_buddy_free_list(struct drm_buddy *mm,
+- struct list_head *objects,
+- bool mark_clear,
+- bool mark_dirty)
+-{
+- struct drm_buddy_block *block, *on;
+-
+- WARN_ON(mark_dirty && mark_clear);
+-
+- list_for_each_entry_safe(block, on, objects, link) {
+- if (mark_clear)
+- mark_cleared(block);
+- else if (mark_dirty)
+- clear_reset(block);
+- drm_buddy_free_block(mm, block);
+- cond_resched();
+- }
+- INIT_LIST_HEAD(objects);
+-}
+-
+-static void drm_buddy_free_list_internal(struct drm_buddy *mm,
+- struct list_head *objects)
+-{
+- /*
+- * Don't touch the clear/dirty bit, since allocation is still internal
+- * at this point. For example we might have just failed part of the
+- * allocation.
+- */
+- __drm_buddy_free_list(mm, objects, false, false);
+-}
+-
+-/**
+- * drm_buddy_free_list - free blocks
+- *
+- * @mm: DRM buddy manager
+- * @objects: input list head to free blocks
+- * @flags: optional flags like DRM_BUDDY_CLEARED
+- */
+-void drm_buddy_free_list(struct drm_buddy *mm,
+- struct list_head *objects,
+- unsigned int flags)
+-{
+- bool mark_clear = flags & DRM_BUDDY_CLEARED;
+-
+- __drm_buddy_free_list(mm, objects, mark_clear, !mark_clear);
+-}
+-EXPORT_SYMBOL(drm_buddy_free_list);
+-
+-static bool block_incompatible(struct drm_buddy_block *block, unsigned int flags)
+-{
+- bool needs_clear = flags & DRM_BUDDY_CLEAR_ALLOCATION;
+-
+- return needs_clear != drm_buddy_block_is_clear(block);
+-}
+-
+-static struct drm_buddy_block *
+-__alloc_range_bias(struct drm_buddy *mm,
+- u64 start, u64 end,
+- unsigned int order,
+- unsigned long flags,
+- bool fallback)
+-{
+- u64 req_size = mm->chunk_size << order;
+- struct drm_buddy_block *block;
+- struct drm_buddy_block *buddy;
+- LIST_HEAD(dfs);
+- int err;
+- int i;
+-
+- end = end - 1;
+-
+- for (i = 0; i < mm->n_roots; ++i)
+- list_add_tail(&mm->roots[i]->tmp_link, &dfs);
+-
+- do {
+- u64 block_start;
+- u64 block_end;
+-
+- block = list_first_entry_or_null(&dfs,
+- struct drm_buddy_block,
+- tmp_link);
+- if (!block)
+- break;
+-
+- list_del(&block->tmp_link);
+-
+- if (drm_buddy_block_order(block) < order)
+- continue;
+-
+- block_start = drm_buddy_block_offset(block);
+- block_end = block_start + drm_buddy_block_size(mm, block) - 1;
+-
+- if (!overlaps(start, end, block_start, block_end))
+- continue;
+-
+- if (drm_buddy_block_is_allocated(block))
+- continue;
+-
+- if (block_start < start || block_end > end) {
+- u64 adjusted_start = max(block_start, start);
+- u64 adjusted_end = min(block_end, end);
+-
+- if (round_down(adjusted_end + 1, req_size) <=
+- round_up(adjusted_start, req_size))
+- continue;
+- }
+-
+- if (!fallback && block_incompatible(block, flags))
+- continue;
+-
+- if (contains(start, end, block_start, block_end) &&
+- order == drm_buddy_block_order(block)) {
+- /*
+- * Find the free block within the range.
+- */
+- if (drm_buddy_block_is_free(block))
+- return block;
+-
+- continue;
+- }
+-
+- if (!drm_buddy_block_is_split(block)) {
+- err = split_block(mm, block);
+- if (unlikely(err))
+- goto err_undo;
+- }
+-
+- list_add(&block->right->tmp_link, &dfs);
+- list_add(&block->left->tmp_link, &dfs);
+- } while (1);
+-
+- return ERR_PTR(-ENOSPC);
+-
+-err_undo:
+- /*
+- * We really don't want to leave around a bunch of split blocks, since
+- * bigger is better, so make sure we merge everything back before we
+- * free the allocated blocks.
+- */
+- buddy = __get_buddy(block);
+- if (buddy &&
+- (drm_buddy_block_is_free(block) &&
+- drm_buddy_block_is_free(buddy)))
+- __drm_buddy_free(mm, block, false);
+- return ERR_PTR(err);
+-}
+-
+-static struct drm_buddy_block *
+-__drm_buddy_alloc_range_bias(struct drm_buddy *mm,
+- u64 start, u64 end,
+- unsigned int order,
+- unsigned long flags)
+-{
+- struct drm_buddy_block *block;
+- bool fallback = false;
+-
+- block = __alloc_range_bias(mm, start, end, order,
+- flags, fallback);
+- if (IS_ERR(block))
+- return __alloc_range_bias(mm, start, end, order,
+- flags, !fallback);
+-
+- return block;
+-}
+-
+-static struct drm_buddy_block *
+-get_maxblock(struct drm_buddy *mm,
+- unsigned int order,
+- enum drm_buddy_free_tree tree)
+-{
+- struct drm_buddy_block *max_block = NULL, *block = NULL;
+- struct rb_root *root;
+- unsigned int i;
+-
+- for (i = order; i <= mm->max_order; ++i) {
+- root = &mm->free_trees[tree][i];
+- block = rbtree_last_free_block(root);
+- if (!block)
+- continue;
+-
+- if (!max_block) {
+- max_block = block;
+- continue;
+- }
+-
+- if (drm_buddy_block_offset(block) >
+- drm_buddy_block_offset(max_block)) {
+- max_block = block;
+- }
+- }
+-
+- return max_block;
+-}
+-
+-static struct drm_buddy_block *
+-alloc_from_freetree(struct drm_buddy *mm,
+- unsigned int order,
+- unsigned long flags)
+-{
+- struct drm_buddy_block *block = NULL;
+- struct rb_root *root;
+- enum drm_buddy_free_tree tree;
+- unsigned int tmp;
+- int err;
+-
+- tree = (flags & DRM_BUDDY_CLEAR_ALLOCATION) ?
+- DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE;
+-
+- if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) {
+- block = get_maxblock(mm, order, tree);
+- if (block)
+- /* Store the obtained block order */
+- tmp = drm_buddy_block_order(block);
+- } else {
+- for (tmp = order; tmp <= mm->max_order; ++tmp) {
+- /* Get RB tree root for this order and tree */
+- root = &mm->free_trees[tree][tmp];
+- block = rbtree_last_free_block(root);
+- if (block)
+- break;
+- }
+- }
+-
+- if (!block) {
+- /* Try allocating from the other tree */
+- tree = (tree == DRM_BUDDY_CLEAR_TREE) ?
+- DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE;
+-
+- for (tmp = order; tmp <= mm->max_order; ++tmp) {
+- root = &mm->free_trees[tree][tmp];
+- block = rbtree_last_free_block(root);
+- if (block)
+- break;
+- }
+-
+- if (!block)
+- return ERR_PTR(-ENOSPC);
+- }
+-
+- BUG_ON(!drm_buddy_block_is_free(block));
+-
+- while (tmp != order) {
+- err = split_block(mm, block);
+- if (unlikely(err))
+- goto err_undo;
+-
+- block = block->right;
+- tmp--;
+- }
+- return block;
+-
+-err_undo:
+- if (tmp != order)
+- __drm_buddy_free(mm, block, false);
+- return ERR_PTR(err);
+-}
+-
+-static int __alloc_range(struct drm_buddy *mm,
+- struct list_head *dfs,
+- u64 start, u64 size,
+- struct list_head *blocks,
+- u64 *total_allocated_on_err)
+-{
+- struct drm_buddy_block *block;
+- struct drm_buddy_block *buddy;
+- u64 total_allocated = 0;
+- LIST_HEAD(allocated);
+- u64 end;
+- int err;
+-
+- end = start + size - 1;
+-
+- do {
+- u64 block_start;
+- u64 block_end;
+-
+- block = list_first_entry_or_null(dfs,
+- struct drm_buddy_block,
+- tmp_link);
+- if (!block)
+- break;
+-
+- list_del(&block->tmp_link);
+-
+- block_start = drm_buddy_block_offset(block);
+- block_end = block_start + drm_buddy_block_size(mm, block) - 1;
+-
+- if (!overlaps(start, end, block_start, block_end))
+- continue;
+-
+- if (drm_buddy_block_is_allocated(block)) {
+- err = -ENOSPC;
+- goto err_free;
+- }
+-
+- if (contains(start, end, block_start, block_end)) {
+- if (drm_buddy_block_is_free(block)) {
+- mark_allocated(mm, block);
+- total_allocated += drm_buddy_block_size(mm, block);
+- mm->avail -= drm_buddy_block_size(mm, block);
+- if (drm_buddy_block_is_clear(block))
+- mm->clear_avail -= drm_buddy_block_size(mm, block);
+- list_add_tail(&block->link, &allocated);
+- continue;
+- } else if (!mm->clear_avail) {
+- err = -ENOSPC;
+- goto err_free;
+- }
+- }
+-
+- if (!drm_buddy_block_is_split(block)) {
+- err = split_block(mm, block);
+- if (unlikely(err))
+- goto err_undo;
+- }
+-
+- list_add(&block->right->tmp_link, dfs);
+- list_add(&block->left->tmp_link, dfs);
+- } while (1);
+-
+- if (total_allocated < size) {
+- err = -ENOSPC;
+- goto err_free;
+- }
+-
+- list_splice_tail(&allocated, blocks);
+-
+- return 0;
+-
+-err_undo:
+- /*
+- * We really don't want to leave around a bunch of split blocks, since
+- * bigger is better, so make sure we merge everything back before we
+- * free the allocated blocks.
+- */
+- buddy = __get_buddy(block);
+- if (buddy &&
+- (drm_buddy_block_is_free(block) &&
+- drm_buddy_block_is_free(buddy)))
+- __drm_buddy_free(mm, block, false);
+-
+-err_free:
+- if (err == -ENOSPC && total_allocated_on_err) {
+- list_splice_tail(&allocated, blocks);
+- *total_allocated_on_err = total_allocated;
+- } else {
+- drm_buddy_free_list_internal(mm, &allocated);
+- }
+-
+- return err;
+-}
+-
+-static int __drm_buddy_alloc_range(struct drm_buddy *mm,
+- u64 start,
+- u64 size,
+- u64 *total_allocated_on_err,
+- struct list_head *blocks)
+-{
+- LIST_HEAD(dfs);
+- int i;
+-
+- for (i = 0; i < mm->n_roots; ++i)
+- list_add_tail(&mm->roots[i]->tmp_link, &dfs);
+-
+- return __alloc_range(mm, &dfs, start, size,
+- blocks, total_allocated_on_err);
+-}
+-
+-static int __alloc_contig_try_harder(struct drm_buddy *mm,
+- u64 size,
+- u64 min_block_size,
+- struct list_head *blocks)
+-{
+- u64 rhs_offset, lhs_offset, lhs_size, filled;
+- struct drm_buddy_block *block;
+- unsigned int tree, order;
+- LIST_HEAD(blocks_lhs);
+- unsigned long pages;
+- u64 modify_size;
+- int err;
+-
+- modify_size = rounddown_pow_of_two(size);
+- pages = modify_size >> ilog2(mm->chunk_size);
+- order = fls(pages) - 1;
+- if (order == 0)
+- return -ENOSPC;
+-
+- for_each_free_tree(tree) {
+- struct rb_root *root;
+- struct rb_node *iter;
+-
+- root = &mm->free_trees[tree][order];
+- if (rbtree_is_empty(root))
+- continue;
+-
+- iter = rb_last(root);
+- while (iter) {
+- block = rbtree_get_free_block(iter);
+-
+- /* Allocate blocks traversing RHS */
+- rhs_offset = drm_buddy_block_offset(block);
+- err = __drm_buddy_alloc_range(mm, rhs_offset, size,
+- &filled, blocks);
+- if (!err || err != -ENOSPC)
+- return err;
+-
+- lhs_size = max((size - filled), min_block_size);
+- if (!IS_ALIGNED(lhs_size, min_block_size))
+- lhs_size = round_up(lhs_size, min_block_size);
+-
+- /* Allocate blocks traversing LHS */
+- lhs_offset = drm_buddy_block_offset(block) - lhs_size;
+- err = __drm_buddy_alloc_range(mm, lhs_offset, lhs_size,
+- NULL, &blocks_lhs);
+- if (!err) {
+- list_splice(&blocks_lhs, blocks);
+- return 0;
+- } else if (err != -ENOSPC) {
+- drm_buddy_free_list_internal(mm, blocks);
+- return err;
+- }
+- /* Free blocks for the next iteration */
+- drm_buddy_free_list_internal(mm, blocks);
+-
+- iter = rb_prev(iter);
+- }
+- }
+-
+- return -ENOSPC;
+-}
+-
+-/**
+- * drm_buddy_block_trim - free unused pages
+- *
+- * @mm: DRM buddy manager
+- * @start: start address to begin the trimming.
+- * @new_size: original size requested
+- * @blocks: Input and output list of allocated blocks.
+- * MUST contain single block as input to be trimmed.
+- * On success will contain the newly allocated blocks
+- * making up the @new_size. Blocks always appear in
+- * ascending order
+- *
+- * For contiguous allocation, we round up the size to the nearest
+- * power of two value, drivers consume *actual* size, so remaining
+- * portions are unused and can be optionally freed with this function
+- *
+- * Returns:
+- * 0 on success, error code on failure.
+- */
+-int drm_buddy_block_trim(struct drm_buddy *mm,
+- u64 *start,
+- u64 new_size,
+- struct list_head *blocks)
+-{
+- struct drm_buddy_block *parent;
+- struct drm_buddy_block *block;
+- u64 block_start, block_end;
+- LIST_HEAD(dfs);
+- u64 new_start;
+- int err;
+-
+- if (!list_is_singular(blocks))
+- return -EINVAL;
+-
+- block = list_first_entry(blocks,
+- struct drm_buddy_block,
+- link);
+-
+- block_start = drm_buddy_block_offset(block);
+- block_end = block_start + drm_buddy_block_size(mm, block);
+-
+- if (WARN_ON(!drm_buddy_block_is_allocated(block)))
+- return -EINVAL;
+-
+- if (new_size > drm_buddy_block_size(mm, block))
+- return -EINVAL;
+-
+- if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size))
+- return -EINVAL;
+-
+- if (new_size == drm_buddy_block_size(mm, block))
+- return 0;
+-
+- new_start = block_start;
+- if (start) {
+- new_start = *start;
+-
+- if (new_start < block_start)
+- return -EINVAL;
+-
+- if (!IS_ALIGNED(new_start, mm->chunk_size))
+- return -EINVAL;
+-
+- if (range_overflows(new_start, new_size, block_end))
+- return -EINVAL;
+- }
+-
+- list_del(&block->link);
+- mark_free(mm, block);
+- mm->avail += drm_buddy_block_size(mm, block);
+- if (drm_buddy_block_is_clear(block))
+- mm->clear_avail += drm_buddy_block_size(mm, block);
+-
+- /* Prevent recursively freeing this node */
+- parent = block->parent;
+- block->parent = NULL;
+-
+- list_add(&block->tmp_link, &dfs);
+- err = __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL);
+- if (err) {
+- mark_allocated(mm, block);
+- mm->avail -= drm_buddy_block_size(mm, block);
+- if (drm_buddy_block_is_clear(block))
+- mm->clear_avail -= drm_buddy_block_size(mm, block);
+- list_add(&block->link, blocks);
+- }
+-
+- block->parent = parent;
+- return err;
+-}
+-EXPORT_SYMBOL(drm_buddy_block_trim);
+-
+-static struct drm_buddy_block *
+-__drm_buddy_alloc_blocks(struct drm_buddy *mm,
+- u64 start, u64 end,
+- unsigned int order,
+- unsigned long flags)
+-{
+- if (flags & DRM_BUDDY_RANGE_ALLOCATION)
+- /* Allocate traversing within the range */
+- return __drm_buddy_alloc_range_bias(mm, start, end,
+- order, flags);
+- else
+- /* Allocate from freetree */
+- return alloc_from_freetree(mm, order, flags);
+-}
+-
+-/**
+- * drm_buddy_alloc_blocks - allocate power-of-two blocks
+- *
+- * @mm: DRM buddy manager to allocate from
+- * @start: start of the allowed range for this block
+- * @end: end of the allowed range for this block
+- * @size: size of the allocation in bytes
+- * @min_block_size: alignment of the allocation
+- * @blocks: output list head to add allocated blocks
+- * @flags: DRM_BUDDY_*_ALLOCATION flags
+- *
+- * alloc_range_bias() called on range limitations, which traverses
+- * the tree and returns the desired block.
+- *
+- * alloc_from_freetree() called when *no* range restrictions
+- * are enforced, which picks the block from the freetree.
+- *
+- * Returns:
+- * 0 on success, error code on failure.
+- */
+-int drm_buddy_alloc_blocks(struct drm_buddy *mm,
+- u64 start, u64 end, u64 size,
+- u64 min_block_size,
+- struct list_head *blocks,
+- unsigned long flags)
+-{
+- struct drm_buddy_block *block = NULL;
+- u64 original_size, original_min_size;
+- unsigned int min_order, order;
+- LIST_HEAD(allocated);
+- unsigned long pages;
+- int err;
+-
+- if (size < mm->chunk_size)
+- return -EINVAL;
+-
+- if (min_block_size < mm->chunk_size)
+- return -EINVAL;
+-
+- if (!is_power_of_2(min_block_size))
+- return -EINVAL;
+-
+- if (!IS_ALIGNED(start | end | size, mm->chunk_size))
+- return -EINVAL;
+-
+- if (end > mm->size)
+- return -EINVAL;
+-
+- if (range_overflows(start, size, mm->size))
+- return -EINVAL;
+-
+- /* Actual range allocation */
+- if (start + size == end) {
+- if (!IS_ALIGNED(start | end, min_block_size))
+- return -EINVAL;
+-
+- return __drm_buddy_alloc_range(mm, start, size, NULL, blocks);
+- }
+-
+- original_size = size;
+- original_min_size = min_block_size;
+-
+- /* Roundup the size to power of 2 */
+- if (flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) {
+- size = roundup_pow_of_two(size);
+- min_block_size = size;
+- /* Align size value to min_block_size */
+- } else if (!IS_ALIGNED(size, min_block_size)) {
+- size = round_up(size, min_block_size);
+- }
+-
+- pages = size >> ilog2(mm->chunk_size);
+- order = fls(pages) - 1;
+- min_order = ilog2(min_block_size) - ilog2(mm->chunk_size);
+-
+- if (order > mm->max_order || size > mm->size) {
+- if ((flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) &&
+- !(flags & DRM_BUDDY_RANGE_ALLOCATION))
+- return __alloc_contig_try_harder(mm, original_size,
+- original_min_size, blocks);
+-
+- return -EINVAL;
+- }
+-
+- do {
+- order = min(order, (unsigned int)fls(pages) - 1);
+- BUG_ON(order > mm->max_order);
+- BUG_ON(order < min_order);
+-
+- do {
+- block = __drm_buddy_alloc_blocks(mm, start,
+- end,
+- order,
+- flags);
+- if (!IS_ERR(block))
+- break;
+-
+- if (order-- == min_order) {
+- /* Try allocation through force merge method */
+- if (mm->clear_avail &&
+- !__force_merge(mm, start, end, min_order)) {
+- block = __drm_buddy_alloc_blocks(mm, start,
+- end,
+- min_order,
+- flags);
+- if (!IS_ERR(block)) {
+- order = min_order;
+- break;
+- }
+- }
+-
+- /*
+- * Try contiguous block allocation through
+- * try harder method.
+- */
+- if (flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION &&
+- !(flags & DRM_BUDDY_RANGE_ALLOCATION))
+- return __alloc_contig_try_harder(mm,
+- original_size,
+- original_min_size,
+- blocks);
+- err = -ENOSPC;
+- goto err_free;
+- }
+- } while (1);
+-
+- mark_allocated(mm, block);
+- mm->avail -= drm_buddy_block_size(mm, block);
+- if (drm_buddy_block_is_clear(block))
+- mm->clear_avail -= drm_buddy_block_size(mm, block);
+- kmemleak_update_trace(block);
+- list_add_tail(&block->link, &allocated);
+-
+- pages -= BIT(order);
+-
+- if (!pages)
+- break;
+- } while (1);
+-
+- /* Trim the allocated block to the required size */
+- if (!(flags & DRM_BUDDY_TRIM_DISABLE) &&
+- original_size != size) {
+- struct list_head *trim_list;
+- LIST_HEAD(temp);
+- u64 trim_size;
+-
+- trim_list = &allocated;
+- trim_size = original_size;
+-
+- if (!list_is_singular(&allocated)) {
+- block = list_last_entry(&allocated, typeof(*block), link);
+- list_move(&block->link, &temp);
+- trim_list = &temp;
+- trim_size = drm_buddy_block_size(mm, block) -
+- (size - original_size);
+- }
+-
+- drm_buddy_block_trim(mm,
+- NULL,
+- trim_size,
+- trim_list);
+-
+- if (!list_empty(&temp))
+- list_splice_tail(trim_list, &allocated);
+- }
+-
+- list_splice_tail(&allocated, blocks);
+- return 0;
+-
+-err_free:
+- drm_buddy_free_list_internal(mm, &allocated);
+- return err;
+-}
+-EXPORT_SYMBOL(drm_buddy_alloc_blocks);
++#include <drm/drm_print.h>
+
+ /**
+ * drm_buddy_block_print - print block information
+@@ -1257,12 +21,12 @@ EXPORT_SYMBOL(drm_buddy_alloc_blocks);
+ * @block: DRM buddy block
+ * @p: DRM printer to use
+ */
+-void drm_buddy_block_print(struct drm_buddy *mm,
+- struct drm_buddy_block *block,
++void drm_buddy_block_print(struct gpu_buddy *mm,
++ struct gpu_buddy_block *block,
+ struct drm_printer *p)
+ {
+- u64 start = drm_buddy_block_offset(block);
+- u64 size = drm_buddy_block_size(mm, block);
++ u64 start = gpu_buddy_block_offset(block);
++ u64 size = gpu_buddy_block_size(mm, block);
+
+ drm_printf(p, "%#018llx-%#018llx: %llu\n", start, start + size, size);
+ }
+@@ -1274,7 +38,7 @@ EXPORT_SYMBOL(drm_buddy_block_print);
+ * @mm: DRM buddy manager
+ * @p: DRM printer to use
+ */
+-void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
++void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p)
+ {
+ int order;
+
+@@ -1282,7 +46,7 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
+ mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
+
+ for (order = mm->max_order; order >= 0; order--) {
+- struct drm_buddy_block *block, *tmp;
++ struct gpu_buddy_block *block, *tmp;
+ struct rb_root *root;
+ u64 count = 0, free;
+ unsigned int tree;
+@@ -1291,7 +55,7 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
+ root = &mm->free_trees[tree][order];
+
+ rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
+- BUG_ON(!drm_buddy_block_is_free(block));
++ BUG_ON(!gpu_buddy_block_is_free(block));
+ count++;
+ }
+ }
+@@ -1309,22 +73,5 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
+ }
+ EXPORT_SYMBOL(drm_buddy_print);
+
+-static void drm_buddy_module_exit(void)
+-{
+- kmem_cache_destroy(slab_blocks);
+-}
+-
+-static int __init drm_buddy_module_init(void)
+-{
+- slab_blocks = KMEM_CACHE(drm_buddy_block, 0);
+- if (!slab_blocks)
+- return -ENOMEM;
+-
+- return 0;
+-}
+-
+-module_init(drm_buddy_module_init);
+-module_exit(drm_buddy_module_exit);
+-
+-MODULE_DESCRIPTION("DRM Buddy Allocator");
++MODULE_DESCRIPTION("DRM-specific GPU Buddy Allocator Print Helpers");
+ MODULE_LICENSE("Dual MIT/GPL");
+diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
+index 994afa5a0ffb52..6c1a49fe9fc666 100644
+--- a/drivers/gpu/drm/drm_connector.c
++++ b/drivers/gpu/drm/drm_connector.c
+@@ -3393,7 +3393,7 @@ EXPORT_SYMBOL(drm_mode_put_tile_group);
+ /**
+ * drm_mode_get_tile_group - get a reference to an existing tile group
+ * @dev: DRM device
+- * @topology: 8-bytes unique per monitor.
++ * @topology_id: 9-byte unique ID per monitor.
+ *
+ * Use the unique bytes to get a reference to an existing tile group.
+ *
+@@ -3401,14 +3401,14 @@ EXPORT_SYMBOL(drm_mode_put_tile_group);
+ * tile group or NULL if not found.
+ */
+ struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev,
+- const char topology[8])
++ const char topology_id[9])
+ {
+ struct drm_tile_group *tg;
+ int id;
+
+ mutex_lock(&dev->mode_config.idr_mutex);
+ idr_for_each_entry(&dev->mode_config.tile_idr, tg, id) {
+- if (!memcmp(tg->group_data, topology, 8)) {
++ if (!memcmp(tg->group_data, topology_id, sizeof(tg->group_data))) {
+ if (!kref_get_unless_zero(&tg->refcount))
+ tg = NULL;
+ mutex_unlock(&dev->mode_config.idr_mutex);
+@@ -3423,7 +3423,7 @@ EXPORT_SYMBOL(drm_mode_get_tile_group);
+ /**
+ * drm_mode_create_tile_group - create a tile group from a displayid description
+ * @dev: DRM device
+- * @topology: 8-bytes unique per monitor.
++ * @topology_id: 9-byte unique ID per monitor.
+ *
+ * Create a tile group for the unique monitor, and get a unique
+ * identifier for the tile group.
+@@ -3432,7 +3432,7 @@ EXPORT_SYMBOL(drm_mode_get_tile_group);
+ * new tile group or NULL.
+ */
+ struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
+- const char topology[8])
++ const char topology_id[9])
+ {
+ struct drm_tile_group *tg;
+ int ret;
+@@ -3442,7 +3442,7 @@ struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
+ return NULL;
+
+ kref_init(&tg->refcount);
+- memcpy(tg->group_data, topology, 8);
++ memcpy(tg->group_data, topology_id, sizeof(tg->group_data));
+ tg->dev = dev;
+
+ mutex_lock(&dev->mode_config.idr_mutex);
+diff --git a/drivers/gpu/drm/drm_displayid_internal.h b/drivers/gpu/drm/drm_displayid_internal.h
+index 01ae9812340c70..49a1cea6250181 100644
+--- a/drivers/gpu/drm/drm_displayid_internal.h
++++ b/drivers/gpu/drm/drm_displayid_internal.h
+@@ -108,7 +108,7 @@ struct displayid_tiled_block {
+ u8 topo[3];
+ u8 tile_size[4];
+ u8 tile_pixel_bezel[5];
+- u8 topology_id[8];
++ u8 topology_id[9];
+ } __packed;
+
+ struct displayid_detailed_timings_1 {
+diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
+index bed485374ab0b5..4793d460892b64 100644
+--- a/drivers/gpu/drm/i915/display/intel_bios.c
++++ b/drivers/gpu/drm/i915/display/intel_bios.c
+@@ -620,6 +620,21 @@ get_lfp_data_tail(const struct bdb_lfp_data *data,
+ return NULL;
+ }
+
++static bool is_panel_type_valid(int panel_type)
++{
++ return panel_type >= 0 && panel_type < 16;
++}
++
++static bool is_panel_type_pnp(int panel_type)
++{
++ return panel_type == 0xff;
++}
++
++static bool is_panel_type_valid_or_pnp(int panel_type)
++{
++ return is_panel_type_valid(panel_type) || is_panel_type_pnp(panel_type);
++}
++
+ static int opregion_get_panel_type(struct intel_display *display,
+ const struct intel_bios_encoder_data *devdata,
+ const struct drm_edid *drm_edid, bool use_fallback)
+@@ -637,15 +652,21 @@ static int vbt_get_panel_type(struct intel_display *display,
+ if (!lfp_options)
+ return -1;
+
+- if (lfp_options->panel_type > 0xf &&
+- lfp_options->panel_type != 0xff) {
++ if (!is_panel_type_valid_or_pnp(lfp_options->panel_type)) {
+ drm_dbg_kms(display->drm, "Invalid VBT panel type 0x%x\n",
+ lfp_options->panel_type);
+ return -1;
+ }
+
+- if (devdata && devdata->child.handle == DEVICE_HANDLE_LFP2)
++ if (devdata && devdata->child.handle == DEVICE_HANDLE_LFP2) {
++ if (!is_panel_type_valid_or_pnp(lfp_options->panel_type2)) {
++ drm_dbg_kms(display->drm, "Invalid VBT panel type 2 0x%x\n",
++ lfp_options->panel_type2);
++ return -1;
++ }
++
+ return lfp_options->panel_type2;
++ }
+
+ drm_WARN_ON(display->drm,
+ devdata && devdata->child.handle != DEVICE_HANDLE_LFP1);
+@@ -759,13 +780,12 @@ static int get_panel_type(struct intel_display *display,
+ panel_types[i].name, panel_types[i].panel_type);
+ }
+
+- if (panel_types[PANEL_TYPE_OPREGION].panel_type >= 0)
++ if (is_panel_type_valid(panel_types[PANEL_TYPE_OPREGION].panel_type))
+ i = PANEL_TYPE_OPREGION;
+- else if (panel_types[PANEL_TYPE_VBT].panel_type == 0xff &&
+- panel_types[PANEL_TYPE_PNPID].panel_type >= 0)
++ else if (is_panel_type_pnp(panel_types[PANEL_TYPE_VBT].panel_type) &&
++ is_panel_type_valid(panel_types[PANEL_TYPE_PNPID].panel_type))
+ i = PANEL_TYPE_PNPID;
+- else if (panel_types[PANEL_TYPE_VBT].panel_type != 0xff &&
+- panel_types[PANEL_TYPE_VBT].panel_type >= 0)
++ else if (is_panel_type_valid(panel_types[PANEL_TYPE_VBT].panel_type))
+ i = PANEL_TYPE_VBT;
+ else
+ i = PANEL_TYPE_FALLBACK;
+diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
+index c0543c35cd6ac4..d900c8125af057 100644
+--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
++++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
+@@ -613,6 +613,7 @@ set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base,
+ return -EINVAL;
+ }
+
++ slot = array_index_nospec(slot, set->num_engines);
+ if (set->engines[slot].type != I915_GEM_ENGINE_TYPE_INVALID) {
+ drm_dbg(&i915->drm,
+ "Invalid placement[%d], already occupied\n", slot);
+@@ -770,8 +771,8 @@ static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv,
+ struct intel_engine_cs *engine;
+
+ if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) {
+- kfree(set.engines);
+- return -EFAULT;
++ err = -EFAULT;
++ goto err;
+ }
+
+ memset(&set.engines[n], 0, sizeof(set.engines[n]));
+@@ -787,8 +788,8 @@ static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv,
+ drm_dbg(&i915->drm,
+ "Invalid engine[%d]: { class:%d, instance:%d }\n",
+ n, ci.engine_class, ci.engine_instance);
+- kfree(set.engines);
+- return -ENOENT;
++ err = -ENOENT;
++ goto err;
+ }
+
+ set.engines[n].type = I915_GEM_ENGINE_TYPE_PHYSICAL;
+@@ -801,15 +802,21 @@ static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv,
+ set_proto_ctx_engines_extensions,
+ ARRAY_SIZE(set_proto_ctx_engines_extensions),
+ &set);
+- if (err) {
+- kfree(set.engines);
+- return err;
+- }
++ if (err)
++ goto err_extensions;
+
+ pc->num_user_engines = set.num_engines;
+ pc->user_engines = set.engines;
+
+ return 0;
++
++err_extensions:
++ for (n = 0; n < set.num_engines; n++)
++ kfree(set.engines[n].siblings);
++err:
++ kfree(set.engines);
++
++ return err;
+ }
+
+ static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv,
+@@ -851,7 +858,7 @@ static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv,
+ pe = &pc->user_engines[idx];
+
+ /* Only render engine supports RPCS configuration. */
+- if (pe->engine->class != RENDER_CLASS)
++ if (!pe->engine || pe->engine->class != RENDER_CLASS)
+ return -EINVAL;
+
+ sseu = &pe->sseu;
+diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+index ed610931f15110..ecb10c2db5a066 100644
+--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
++++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+@@ -7,7 +7,7 @@
+
+ #include <drm/ttm/ttm_placement.h>
+ #include <drm/ttm/ttm_tt.h>
+-#include <drm/drm_buddy.h>
++#include <linux/gpu_buddy.h>
+
+ #include "i915_drv.h"
+ #include "i915_ttm_buddy_manager.h"
+diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c
+index 833987015b8bbc..ed42b9d05a07d5 100644
+--- a/drivers/gpu/drm/i915/gt/intel_engine_user.c
++++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c
+@@ -257,7 +257,7 @@ void intel_engines_driver_register(struct drm_i915_private *i915)
+ p = &prev->rb_right;
+ }
+
+- if (IS_ENABLED(CONFIG_DRM_I915_SELFTESTS) &&
++ if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST) &&
+ IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
+ struct intel_engine_cs *engine;
+ unsigned int isolation;
+diff --git a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
+index 33351deeea4f0b..07eaf71955c447 100644
+--- a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
++++ b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
+@@ -16,9 +16,9 @@ static int cmp_u64(const void *A, const void *B)
+ {
+ const u64 *a = A, *b = B;
+
+- if (a < b)
++ if (*a < *b)
+ return -1;
+- else if (a > b)
++ else if (*a > *b)
+ return 1;
+ else
+ return 0;
+@@ -28,9 +28,9 @@ static int cmp_u32(const void *A, const void *B)
+ {
+ const u32 *a = A, *b = B;
+
+- if (a < b)
++ if (*a < *b)
+ return -1;
+- else if (a > b)
++ else if (*a > *b)
+ return 1;
+ else
+ return 0;
+diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
+index 5ec293011d9902..40f84d547274a4 100644
+--- a/drivers/gpu/drm/i915/i915_active.c
++++ b/drivers/gpu/drm/i915/i915_active.c
+@@ -319,7 +319,7 @@ active_instance(struct i915_active *ref, u64 idx)
+ */
+ node = kmem_cache_alloc(slab_cache, GFP_ATOMIC);
+ if (!node)
+- goto out;
++ goto err;
+
+ __i915_active_fence_init(&node->base, NULL, node_retire);
+ node->ref = ref;
+@@ -333,6 +333,11 @@ out:
+ spin_unlock_irq(&ref->tree_lock);
+
+ return &node->base;
++
++err:
++ spin_unlock_irq(&ref->tree_lock);
++
++ return NULL;
+ }
+
+ void __i915_active_init(struct i915_active *ref,
+diff --git a/drivers/gpu/drm/i915/i915_scatterlist.c b/drivers/gpu/drm/i915/i915_scatterlist.c
+index 4d830740946da3..6a34dae13769e8 100644
+--- a/drivers/gpu/drm/i915/i915_scatterlist.c
++++ b/drivers/gpu/drm/i915/i915_scatterlist.c
+@@ -7,7 +7,7 @@
+ #include "i915_scatterlist.h"
+ #include "i915_ttm_buddy_manager.h"
+
+-#include <drm/drm_buddy.h>
++#include <linux/gpu_buddy.h>
+ #include <drm/drm_mm.h>
+
+ #include <linux/slab.h>
+@@ -167,9 +167,9 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res,
+ struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
+ const u64 size = res->size;
+ const u32 max_segment = round_down(UINT_MAX, page_alignment);
+- struct drm_buddy *mm = bman_res->mm;
++ struct gpu_buddy *mm = bman_res->mm;
+ struct list_head *blocks = &bman_res->blocks;
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ struct i915_refct_sgt *rsgt;
+ struct scatterlist *sg;
+ struct sg_table *st;
+@@ -202,8 +202,8 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res,
+ list_for_each_entry(block, blocks, link) {
+ u64 block_size, offset;
+
+- block_size = min_t(u64, size, drm_buddy_block_size(mm, block));
+- offset = drm_buddy_block_offset(block);
++ block_size = min_t(u64, size, gpu_buddy_block_size(mm, block));
++ offset = gpu_buddy_block_offset(block);
+
+ while (block_size) {
+ u64 len;
+diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
+index 942345548bc323..297e5336590a6b 100644
+--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
++++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
+@@ -8,6 +8,7 @@
+ #include <drm/ttm/ttm_placement.h>
+ #include <drm/ttm/ttm_bo.h>
+
++#include <linux/gpu_buddy.h>
+ #include <drm/drm_buddy.h>
+
+ #include "i915_ttm_buddy_manager.h"
+@@ -16,7 +17,7 @@
+
+ struct i915_ttm_buddy_manager {
+ struct ttm_resource_manager manager;
+- struct drm_buddy mm;
++ struct gpu_buddy mm;
+ struct list_head reserved;
+ struct mutex lock;
+ unsigned long visible_size;
+@@ -38,7 +39,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
+ {
+ struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+ struct i915_ttm_buddy_resource *bman_res;
+- struct drm_buddy *mm = &bman->mm;
++ struct gpu_buddy *mm = &bman->mm;
+ unsigned long n_pages, lpfn;
+ u64 min_page_size;
+ u64 size;
+@@ -57,13 +58,13 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
+ bman_res->mm = mm;
+
+ if (place->flags & TTM_PL_FLAG_TOPDOWN)
+- bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
++ bman_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
+
+ if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
+- bman_res->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
++ bman_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
+
+ if (place->fpfn || lpfn != man->size)
+- bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
++ bman_res->flags |= GPU_BUDDY_RANGE_ALLOCATION;
+
+ GEM_BUG_ON(!bman_res->base.size);
+ size = bman_res->base.size;
+@@ -89,7 +90,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
+ goto err_free_res;
+ }
+
+- err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
++ err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
+ (u64)lpfn << PAGE_SHIFT,
+ (u64)n_pages << PAGE_SHIFT,
+ min_page_size,
+@@ -101,15 +102,15 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
+ if (lpfn <= bman->visible_size) {
+ bman_res->used_visible_size = PFN_UP(bman_res->base.size);
+ } else {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+
+ list_for_each_entry(block, &bman_res->blocks, link) {
+ unsigned long start =
+- drm_buddy_block_offset(block) >> PAGE_SHIFT;
++ gpu_buddy_block_offset(block) >> PAGE_SHIFT;
+
+ if (start < bman->visible_size) {
+ unsigned long end = start +
+- (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
++ (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
+
+ bman_res->used_visible_size +=
+ min(end, bman->visible_size) - start;
+@@ -126,7 +127,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
+ return 0;
+
+ err_free_blocks:
+- drm_buddy_free_list(mm, &bman_res->blocks, 0);
++ gpu_buddy_free_list(mm, &bman_res->blocks, 0);
+ mutex_unlock(&bman->lock);
+ err_free_res:
+ ttm_resource_fini(man, &bman_res->base);
+@@ -141,7 +142,7 @@ static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
+ struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+
+ mutex_lock(&bman->lock);
+- drm_buddy_free_list(&bman->mm, &bman_res->blocks, 0);
++ gpu_buddy_free_list(&bman->mm, &bman_res->blocks, 0);
+ bman->visible_avail += bman_res->used_visible_size;
+ mutex_unlock(&bman->lock);
+
+@@ -156,8 +157,8 @@ static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
+ {
+ struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
+ struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+- struct drm_buddy *mm = &bman->mm;
+- struct drm_buddy_block *block;
++ struct gpu_buddy *mm = &bman->mm;
++ struct gpu_buddy_block *block;
+
+ if (!place->fpfn && !place->lpfn)
+ return true;
+@@ -176,9 +177,9 @@ static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
+ /* Check each drm buddy block individually */
+ list_for_each_entry(block, &bman_res->blocks, link) {
+ unsigned long fpfn =
+- drm_buddy_block_offset(block) >> PAGE_SHIFT;
++ gpu_buddy_block_offset(block) >> PAGE_SHIFT;
+ unsigned long lpfn = fpfn +
+- (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
++ (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
+
+ if (place->fpfn < lpfn && place->lpfn > fpfn)
+ return true;
+@@ -194,8 +195,8 @@ static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
+ {
+ struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
+ struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+- struct drm_buddy *mm = &bman->mm;
+- struct drm_buddy_block *block;
++ struct gpu_buddy *mm = &bman->mm;
++ struct gpu_buddy_block *block;
+
+ if (!place->fpfn && !place->lpfn)
+ return true;
+@@ -209,9 +210,9 @@ static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
+ /* Check each drm buddy block individually */
+ list_for_each_entry(block, &bman_res->blocks, link) {
+ unsigned long fpfn =
+- drm_buddy_block_offset(block) >> PAGE_SHIFT;
++ gpu_buddy_block_offset(block) >> PAGE_SHIFT;
+ unsigned long lpfn = fpfn +
+- (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
++ (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
+
+ if (fpfn < place->fpfn || lpfn > place->lpfn)
+ return false;
+@@ -224,7 +225,7 @@ static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
+ struct drm_printer *printer)
+ {
+ struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+
+ mutex_lock(&bman->lock);
+ drm_printf(printer, "default_page_size: %lluKiB\n",
+@@ -293,7 +294,7 @@ int i915_ttm_buddy_man_init(struct ttm_device *bdev,
+ if (!bman)
+ return -ENOMEM;
+
+- err = drm_buddy_init(&bman->mm, size, chunk_size);
++ err = gpu_buddy_init(&bman->mm, size, chunk_size);
+ if (err)
+ goto err_free_bman;
+
+@@ -333,7 +334,7 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
+ {
+ struct ttm_resource_manager *man = ttm_manager_type(bdev, type);
+ struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+- struct drm_buddy *mm = &bman->mm;
++ struct gpu_buddy *mm = &bman->mm;
+ int ret;
+
+ ttm_resource_manager_set_used(man, false);
+@@ -345,8 +346,8 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
+ ttm_set_driver_manager(bdev, type, NULL);
+
+ mutex_lock(&bman->lock);
+- drm_buddy_free_list(mm, &bman->reserved, 0);
+- drm_buddy_fini(mm);
++ gpu_buddy_free_list(mm, &bman->reserved, 0);
++ gpu_buddy_fini(mm);
+ bman->visible_avail += bman->visible_reserved;
+ WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
+ mutex_unlock(&bman->lock);
+@@ -371,15 +372,15 @@ int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man,
+ u64 start, u64 size)
+ {
+ struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+- struct drm_buddy *mm = &bman->mm;
++ struct gpu_buddy *mm = &bman->mm;
+ unsigned long fpfn = start >> PAGE_SHIFT;
+ unsigned long flags = 0;
+ int ret;
+
+- flags |= DRM_BUDDY_RANGE_ALLOCATION;
++ flags |= GPU_BUDDY_RANGE_ALLOCATION;
+
+ mutex_lock(&bman->lock);
+- ret = drm_buddy_alloc_blocks(mm, start,
++ ret = gpu_buddy_alloc_blocks(mm, start,
+ start + size,
+ size, mm->chunk_size,
+ &bman->reserved,
+diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h
+index d6462071283039..1cff018c1689af 100644
+--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h
++++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h
+@@ -13,7 +13,7 @@
+
+ struct ttm_device;
+ struct ttm_resource_manager;
+-struct drm_buddy;
++struct gpu_buddy;
+
+ /**
+ * struct i915_ttm_buddy_resource
+@@ -33,7 +33,7 @@ struct i915_ttm_buddy_resource {
+ struct list_head blocks;
+ unsigned long flags;
+ unsigned long used_visible_size;
+- struct drm_buddy *mm;
++ struct gpu_buddy *mm;
+ };
+
+ /**
+diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c
+index f08f6674911eec..518887a5584c43 100644
+--- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c
++++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c
+@@ -6,7 +6,7 @@
+ #include <linux/prime_numbers.h>
+ #include <linux/sort.h>
+
+-#include <drm/drm_buddy.h>
++#include <linux/gpu_buddy.h>
+
+ #include "../i915_selftest.h"
+
+@@ -371,7 +371,7 @@ static int igt_mock_splintered_region(void *arg)
+ struct drm_i915_private *i915 = mem->i915;
+ struct i915_ttm_buddy_resource *res;
+ struct drm_i915_gem_object *obj;
+- struct drm_buddy *mm;
++ struct gpu_buddy *mm;
+ unsigned int expected_order;
+ LIST_HEAD(objects);
+ u64 size;
+@@ -455,8 +455,8 @@ static int igt_mock_max_segment(void *arg)
+ struct drm_i915_private *i915 = mem->i915;
+ struct i915_ttm_buddy_resource *res;
+ struct drm_i915_gem_object *obj;
+- struct drm_buddy_block *block;
+- struct drm_buddy *mm;
++ struct gpu_buddy_block *block;
++ struct gpu_buddy *mm;
+ struct list_head *blocks;
+ struct scatterlist *sg;
+ I915_RND_STATE(prng);
+@@ -495,8 +495,8 @@ static int igt_mock_max_segment(void *arg)
+ mm = res->mm;
+ size = 0;
+ list_for_each_entry(block, blocks, link) {
+- if (drm_buddy_block_size(mm, block) > size)
+- size = drm_buddy_block_size(mm, block);
++ if (gpu_buddy_block_size(mm, block) > size)
++ size = gpu_buddy_block_size(mm, block);
+ }
+ if (size < max_segment) {
+ pr_err("%s: Failed to create a huge contiguous block [> %u], largest block %lld\n",
+@@ -535,14 +535,14 @@ static u64 igt_object_mappable_total(struct drm_i915_gem_object *obj)
+ struct intel_memory_region *mr = obj->mm.region;
+ struct i915_ttm_buddy_resource *bman_res =
+ to_ttm_buddy_resource(obj->mm.res);
+- struct drm_buddy *mm = bman_res->mm;
+- struct drm_buddy_block *block;
++ struct gpu_buddy *mm = bman_res->mm;
++ struct gpu_buddy_block *block;
+ u64 total;
+
+ total = 0;
+ list_for_each_entry(block, &bman_res->blocks, link) {
+- u64 start = drm_buddy_block_offset(block);
+- u64 end = start + drm_buddy_block_size(mm, block);
++ u64 start = gpu_buddy_block_offset(block);
++ u64 end = start + gpu_buddy_block_size(mm, block);
+
+ if (start < resource_size(&mr->io))
+ total += min_t(u64, end, resource_size(&mr->io)) - start;
+diff --git a/drivers/gpu/drm/imagination/pvr_context.c b/drivers/gpu/drm/imagination/pvr_context.c
+index 4cb3494c0bb2c6..e330adba75f1cd 100644
+--- a/drivers/gpu/drm/imagination/pvr_context.c
++++ b/drivers/gpu/drm/imagination/pvr_context.c
+@@ -173,22 +173,24 @@ ctx_fw_data_init(void *cpu_ptr, void *priv)
+ /**
+ * pvr_context_destroy_queues() - Destroy all queues attached to a context.
+ * @ctx: Context to destroy queues on.
++ * @cleanup_queue_entity: Whether to cleanup the queue entity e.g. context
++ * creation failure path.
+ *
+ * Should be called when the last reference to a context object is dropped.
+ * It releases all resources attached to the queues bound to this context.
+ */
+-static void pvr_context_destroy_queues(struct pvr_context *ctx)
++static void pvr_context_destroy_queues(struct pvr_context *ctx, bool cleanup_queue_entity)
+ {
+ switch (ctx->type) {
+ case DRM_PVR_CTX_TYPE_RENDER:
+- pvr_queue_destroy(ctx->queues.fragment);
+- pvr_queue_destroy(ctx->queues.geometry);
++ pvr_queue_destroy(ctx->queues.fragment, cleanup_queue_entity);
++ pvr_queue_destroy(ctx->queues.geometry, cleanup_queue_entity);
+ break;
+ case DRM_PVR_CTX_TYPE_COMPUTE:
+- pvr_queue_destroy(ctx->queues.compute);
++ pvr_queue_destroy(ctx->queues.compute, cleanup_queue_entity);
+ break;
+ case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:
+- pvr_queue_destroy(ctx->queues.transfer);
++ pvr_queue_destroy(ctx->queues.transfer, cleanup_queue_entity);
+ break;
+ }
+ }
+@@ -252,7 +254,7 @@ static int pvr_context_create_queues(struct pvr_context *ctx,
+ return -EINVAL;
+
+ err_destroy_queues:
+- pvr_context_destroy_queues(ctx);
++ pvr_context_destroy_queues(ctx, true);
+ return err;
+ }
+
+@@ -319,8 +321,8 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co
+ goto err_free_ctx;
+
+ ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle);
+- if (IS_ERR(ctx->vm_ctx)) {
+- err = PTR_ERR(ctx->vm_ctx);
++ if (!ctx->vm_ctx) {
++ err = -EINVAL;
+ goto err_free_ctx;
+ }
+
+@@ -368,7 +370,7 @@ err_destroy_fw_obj:
+ pvr_fw_object_destroy(ctx->fw_obj);
+
+ err_destroy_queues:
+- pvr_context_destroy_queues(ctx);
++ pvr_context_destroy_queues(ctx, true);
+
+ err_free_ctx_data:
+ kfree(ctx->data);
+@@ -394,7 +396,7 @@ pvr_context_release(struct kref *ref_count)
+ spin_unlock(&pvr_dev->ctx_list_lock);
+
+ xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id);
+- pvr_context_destroy_queues(ctx);
++ pvr_context_destroy_queues(ctx, false);
+ pvr_fw_object_destroy(ctx->fw_obj);
+ kfree(ctx->data);
+ pvr_vm_context_put(ctx->vm_ctx);
+diff --git a/drivers/gpu/drm/imagination/pvr_drv.c b/drivers/gpu/drm/imagination/pvr_drv.c
+index fb17196e05f498..ba1acf5977cb5f 100644
+--- a/drivers/gpu/drm/imagination/pvr_drv.c
++++ b/drivers/gpu/drm/imagination/pvr_drv.c
+@@ -1251,14 +1251,13 @@ pvr_set_uobj_array(const struct drm_pvr_obj_array *out, u32 min_stride, u32 obj_
+ if (copy_to_user(out_ptr, in_ptr, cpy_elem_size))
+ return -EFAULT;
+
+- out_ptr += obj_size;
+- in_ptr += out->stride;
+- }
++ if (out->stride > obj_size &&
++ clear_user(out_ptr + cpy_elem_size, out->stride - obj_size)) {
++ return -EFAULT;
++ }
+
+- if (out->stride > obj_size &&
+- clear_user(u64_to_user_ptr(out->array + obj_size),
+- out->stride - obj_size)) {
+- return -EFAULT;
++ out_ptr += out->stride;
++ in_ptr += obj_size;
+ }
+ }
+
+diff --git a/drivers/gpu/drm/imagination/pvr_queue.c b/drivers/gpu/drm/imagination/pvr_queue.c
+index 130473cfdfc9b7..083cb61819852b 100644
+--- a/drivers/gpu/drm/imagination/pvr_queue.c
++++ b/drivers/gpu/drm/imagination/pvr_queue.c
+@@ -179,7 +179,7 @@ static const struct dma_fence_ops pvr_queue_job_fence_ops = {
+
+ /**
+ * to_pvr_queue_job_fence() - Return a pvr_queue_fence object if the fence is
+- * backed by a UFO.
++ * already backed by a UFO.
+ * @f: The dma_fence to turn into a pvr_queue_fence.
+ *
+ * Return:
+@@ -356,6 +356,15 @@ static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count)
+ pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len);
+ }
+
++static bool
++is_paired_job_fence(struct dma_fence *fence, struct pvr_job *job)
++{
++ /* This assumes "fence" is one of "job"'s drm_sched_job::dependencies */
++ return job->type == DRM_PVR_JOB_TYPE_FRAGMENT &&
++ job->paired_job &&
++ &job->paired_job->base.s_fence->scheduled == fence;
++}
++
+ /**
+ * job_count_remaining_native_deps() - Count the number of non-signaled native dependencies.
+ * @job: Job to operate on.
+@@ -371,6 +380,17 @@ static unsigned long job_count_remaining_native_deps(struct pvr_job *job)
+ xa_for_each(&job->base.dependencies, index, fence) {
+ struct pvr_queue_fence *jfence;
+
++ if (is_paired_job_fence(fence, job)) {
++ /*
++ * A fence between paired jobs won't resolve to a pvr_queue_fence (i.e.
++ * be backed by a UFO) until the jobs have been submitted, together.
++ * The submitting code will insert a partial render fence command for this.
++ */
++ WARN_ON(dma_fence_is_signaled(fence));
++ remaining_count++;
++ continue;
++ }
++
+ jfence = to_pvr_queue_job_fence(fence);
+ if (!jfence)
+ continue;
+@@ -468,10 +488,11 @@ pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job)
+ }
+
+ static struct dma_fence *
+-pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job)
++pvr_queue_get_paired_frag_job_dep(struct pvr_job *job)
+ {
+ struct pvr_job *frag_job = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ?
+ job->paired_job : NULL;
++ struct pvr_queue *frag_queue = frag_job ? frag_job->ctx->queues.fragment : NULL;
+ struct dma_fence *f;
+ unsigned long index;
+
+@@ -490,7 +511,10 @@ pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job)
+ return dma_fence_get(f);
+ }
+
+- return frag_job->base.sched->ops->prepare_job(&frag_job->base, &queue->entity);
++ /* Initialize the paired fragment job's done_fence, so we can signal it. */
++ pvr_queue_job_fence_init(frag_job->done_fence, frag_queue);
++
++ return pvr_queue_get_job_cccb_fence(frag_queue, frag_job);
+ }
+
+ /**
+@@ -509,11 +533,6 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job,
+ struct pvr_queue *queue = container_of(s_entity, struct pvr_queue, entity);
+ struct dma_fence *internal_dep = NULL;
+
+- /*
+- * Initialize the done_fence, so we can signal it. This must be done
+- * here because otherwise by the time of run_job() the job will end up
+- * in the pending list without a valid fence.
+- */
+ if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {
+ /*
+ * This will be called on a paired fragment job after being
+@@ -523,18 +542,15 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job,
+ */
+ if (job->paired_job->has_pm_ref)
+ return NULL;
+-
+- /*
+- * In this case we need to use the job's own ctx to initialise
+- * the done_fence. The other steps are done in the ctx of the
+- * paired geometry job.
+- */
+- pvr_queue_job_fence_init(job->done_fence,
+- job->ctx->queues.fragment);
+- } else {
+- pvr_queue_job_fence_init(job->done_fence, queue);
+ }
+
++ /*
++ * Initialize the done_fence, so we can signal it. This must be done
++ * here because otherwise by the time of run_job() the job will end up
++ * in the pending list without a valid fence.
++ */
++ pvr_queue_job_fence_init(job->done_fence, queue);
++
+ /* CCCB fence is used to make sure we have enough space in the CCCB to
+ * submit our commands.
+ */
+@@ -555,7 +571,7 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job,
+
+ /* The paired job fence should come last, when everything else is ready. */
+ if (!internal_dep)
+- internal_dep = pvr_queue_get_paired_frag_job_dep(queue, job);
++ internal_dep = pvr_queue_get_paired_frag_job_dep(job);
+
+ return internal_dep;
+ }
+@@ -630,9 +646,8 @@ static void pvr_queue_submit_job_to_cccb(struct pvr_job *job)
+ if (!jfence)
+ continue;
+
+- /* Skip the partial render fence, we will place it at the end. */
+- if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job &&
+- &job->paired_job->base.s_fence->scheduled == fence)
++ /* This fence will be placed last, as partial render fence. */
++ if (is_paired_job_fence(fence, job))
+ continue;
+
+ if (dma_fence_is_signaled(&jfence->base))
+@@ -1379,11 +1394,12 @@ void pvr_queue_kill(struct pvr_queue *queue)
+ /**
+ * pvr_queue_destroy() - Destroy a queue.
+ * @queue: The queue to destroy.
++ * @cleanup_queue_entity: Whether to cleanup the queue entity.
+ *
+ * Cleanup the queue and free the resources attached to it. Should be
+ * called from the context release function.
+ */
+-void pvr_queue_destroy(struct pvr_queue *queue)
++void pvr_queue_destroy(struct pvr_queue *queue, bool cleanup_queue_entity)
+ {
+ if (!queue)
+ return;
+@@ -1393,7 +1409,8 @@ void pvr_queue_destroy(struct pvr_queue *queue)
+ mutex_unlock(&queue->ctx->pvr_dev->queues.lock);
+
+ drm_sched_fini(&queue->scheduler);
+- drm_sched_entity_fini(&queue->entity);
++ if (cleanup_queue_entity)
++ drm_sched_entity_fini(&queue->entity);
+
+ if (WARN_ON(queue->last_queued_job_scheduled_fence))
+ dma_fence_put(queue->last_queued_job_scheduled_fence);
+diff --git a/drivers/gpu/drm/imagination/pvr_queue.h b/drivers/gpu/drm/imagination/pvr_queue.h
+index 93fe9ac9f58ccc..887f60b5daa62d 100644
+--- a/drivers/gpu/drm/imagination/pvr_queue.h
++++ b/drivers/gpu/drm/imagination/pvr_queue.h
+@@ -158,7 +158,7 @@ struct pvr_queue *pvr_queue_create(struct pvr_context *ctx,
+
+ void pvr_queue_kill(struct pvr_queue *queue);
+
+-void pvr_queue_destroy(struct pvr_queue *queue);
++void pvr_queue_destroy(struct pvr_queue *queue, bool cleanup_queue_entity);
+
+ void pvr_queue_process(struct pvr_queue *queue);
+
+diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c
+index 2896fa7501b1cc..a118bf2b0ea1af 100644
+--- a/drivers/gpu/drm/imagination/pvr_vm.c
++++ b/drivers/gpu/drm/imagination/pvr_vm.c
+@@ -741,6 +741,7 @@ pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,
+
+ pvr_gem_object_get(pvr_obj);
+
++ mutex_lock(&vm_ctx->lock);
+ err = drm_gpuvm_exec_lock(&vm_exec);
+ if (err)
+ goto err_cleanup;
+@@ -750,6 +751,7 @@ pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,
+ drm_gpuvm_exec_unlock(&vm_exec);
+
+ err_cleanup:
++ mutex_unlock(&vm_ctx->lock);
+ pvr_vm_bind_op_fini(&bind_op);
+
+ return err;
+diff --git a/drivers/gpu/drm/lib/drm_random.c b/drivers/gpu/drm/lib/drm_random.c
+deleted file mode 100644
+index 31b5a3e219114f..00000000000000
+--- a/drivers/gpu/drm/lib/drm_random.c
++++ /dev/null
+@@ -1,43 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0
+-#include <linux/bitops.h>
+-#include <linux/kernel.h>
+-#include <linux/random.h>
+-#include <linux/slab.h>
+-#include <linux/types.h>
+-
+-#include "drm_random.h"
+-
+-u32 drm_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state)
+-{
+- return upper_32_bits((u64)prandom_u32_state(state) * ep_ro);
+-}
+-EXPORT_SYMBOL(drm_prandom_u32_max_state);
+-
+-void drm_random_reorder(unsigned int *order, unsigned int count,
+- struct rnd_state *state)
+-{
+- unsigned int i, j;
+-
+- for (i = 0; i < count; ++i) {
+- BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32));
+- j = drm_prandom_u32_max_state(count, state);
+- swap(order[i], order[j]);
+- }
+-}
+-EXPORT_SYMBOL(drm_random_reorder);
+-
+-unsigned int *drm_random_order(unsigned int count, struct rnd_state *state)
+-{
+- unsigned int *order, i;
+-
+- order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
+- if (!order)
+- return order;
+-
+- for (i = 0; i < count; i++)
+- order[i] = i;
+-
+- drm_random_reorder(order, count, state);
+- return order;
+-}
+-EXPORT_SYMBOL(drm_random_order);
+diff --git a/drivers/gpu/drm/lib/drm_random.h b/drivers/gpu/drm/lib/drm_random.h
+deleted file mode 100644
+index 5543bf0474bc4e..00000000000000
+--- a/drivers/gpu/drm/lib/drm_random.h
++++ /dev/null
+@@ -1,28 +0,0 @@
+-/* SPDX-License-Identifier: GPL-2.0 */
+-#ifndef __DRM_RANDOM_H__
+-#define __DRM_RANDOM_H__
+-
+-/* This is a temporary home for a couple of utility functions that should
+- * be transposed to lib/ at the earliest convenience.
+- */
+-
+-#include <linux/random.h>
+-
+-#define DRM_RND_STATE_INITIALIZER(seed__) ({ \
+- struct rnd_state state__; \
+- prandom_seed_state(&state__, (seed__)); \
+- state__; \
+-})
+-
+-#define DRM_RND_STATE(name__, seed__) \
+- struct rnd_state name__ = DRM_RND_STATE_INITIALIZER(seed__)
+-
+-unsigned int *drm_random_order(unsigned int count,
+- struct rnd_state *state);
+-void drm_random_reorder(unsigned int *order,
+- unsigned int count,
+- struct rnd_state *state);
+-u32 drm_prandom_u32_max_state(u32 ep_ro,
+- struct rnd_state *state);
+-
+-#endif /* !__DRM_RANDOM_H__ */
+diff --git a/drivers/gpu/drm/nouveau/nouveau_exec.c b/drivers/gpu/drm/nouveau/nouveau_exec.c
+index a0b5f1b16e8b39..94ad24f2b697ca 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_exec.c
++++ b/drivers/gpu/drm/nouveau/nouveau_exec.c
+@@ -327,10 +327,10 @@ nouveau_exec_ucopy(struct nouveau_exec_job_args *args,
+
+ return 0;
+
+-err_free_pushs:
+- u_free(args->push.s);
+ err_free_ins:
+ u_free(args->in_sync.s);
++err_free_pushs:
++ u_free(args->push.s);
+ return ret;
+ }
+
+diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
+index 48f105239f42d8..f2e332b08eda2f 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c
++++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
+@@ -1708,10 +1708,10 @@ nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args,
+
+ return 0;
+
+-err_free_ops:
+- u_free(args->op.s);
+ err_free_ins:
+ u_free(args->in_sync.s);
++err_free_ops:
++ u_free(args->op.s);
+ return ret;
+ }
+
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c
+index 9b8ca4e898f903..dc82b7d7f27218 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c
+@@ -315,6 +315,7 @@ nvkm_acr_oneinit(struct nvkm_subdev *subdev)
+ i, us, fw);
+ }
+ }
++ nvkm_done(acr->wpr);
+ return -EINVAL;
+ }
+ nvkm_done(acr->wpr);
+diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c
+index cdd6e1c08cebdb..eb4aface8a4ab5 100644
+--- a/drivers/gpu/drm/panthor/panthor_fw.c
++++ b/drivers/gpu/drm/panthor/panthor_fw.c
+@@ -732,6 +732,7 @@ static int panthor_fw_load(struct panthor_device *ptdev)
+ }
+
+ if (hdr.size > iter.size) {
++ ret = -EINVAL;
+ drm_err(&ptdev->base, "Firmware image is truncated\n");
+ goto out;
+ }
+diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
+index 80703417d8a18c..2e7437dbb44137 100644
+--- a/drivers/gpu/drm/radeon/r100.c
++++ b/drivers/gpu/drm/radeon/r100.c
+@@ -906,6 +906,7 @@ struct radeon_fence *r100_copy_blit(struct radeon_device *rdev,
+ {
+ struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
+ struct radeon_fence *fence;
++ uint64_t cur_src_offset, cur_dst_offset;
+ uint32_t cur_pages;
+ uint32_t stride_bytes = RADEON_GPU_PAGE_SIZE;
+ uint32_t pitch;
+@@ -934,6 +935,10 @@ struct radeon_fence *r100_copy_blit(struct radeon_device *rdev,
+ cur_pages = 8191;
+ }
+ num_gpu_pages -= cur_pages;
++ cur_src_offset = src_offset +
++ (uint64_t)num_gpu_pages * RADEON_GPU_PAGE_SIZE;
++ cur_dst_offset = dst_offset +
++ (uint64_t)num_gpu_pages * RADEON_GPU_PAGE_SIZE;
+
+ /* pages are in Y direction - height
+ page width in X direction - width */
+@@ -950,13 +955,13 @@ struct radeon_fence *r100_copy_blit(struct radeon_device *rdev,
+ RADEON_DP_SRC_SOURCE_MEMORY |
+ RADEON_GMC_CLR_CMP_CNTL_DIS |
+ RADEON_GMC_WR_MSK_DIS);
+- radeon_ring_write(ring, (pitch << 22) | (src_offset >> 10));
+- radeon_ring_write(ring, (pitch << 22) | (dst_offset >> 10));
++ radeon_ring_write(ring, (pitch << 22) | (cur_src_offset >> 10));
++ radeon_ring_write(ring, (pitch << 22) | (cur_dst_offset >> 10));
+ radeon_ring_write(ring, (0x1fff) | (0x1fff << 16));
+ radeon_ring_write(ring, 0);
+ radeon_ring_write(ring, (0x1fff) | (0x1fff << 16));
+- radeon_ring_write(ring, num_gpu_pages);
+- radeon_ring_write(ring, num_gpu_pages);
++ radeon_ring_write(ring, 0);
++ radeon_ring_write(ring, 0);
+ radeon_ring_write(ring, cur_pages | (stride_pixels << 16));
+ }
+ radeon_ring_write(ring, PACKET0(RADEON_DSTCACHE_CTLSTAT, 0));
+diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
+index 6cec796dd463f6..a9216a99b2be18 100644
+--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
++++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
+@@ -238,7 +238,7 @@ static int rzg2l_mipi_dsi_dphy_init(struct rzg2l_mipi_dsi *dsi,
+ if (ret < 0)
+ return ret;
+
+- udelay(1);
++ fsleep(1000);
+
+ return 0;
+ }
+diff --git a/drivers/gpu/drm/rockchip/cdn-dp-reg.c b/drivers/gpu/drm/rockchip/cdn-dp-reg.c
+index 33fb4d05c50657..7b16c28ba25a63 100644
+--- a/drivers/gpu/drm/rockchip/cdn-dp-reg.c
++++ b/drivers/gpu/drm/rockchip/cdn-dp-reg.c
+@@ -683,6 +683,8 @@ int cdn_dp_config_video(struct cdn_dp_device *dp)
+ val = div_u64(8 * (symbol + 1), bit_per_pix) - val;
+ val += 2;
+ ret = cdn_dp_reg_write(dp, DP_VC_TABLE(15), val);
++ if (ret)
++ goto err_config_video;
+
+ switch (video->color_depth) {
+ case 6:
+diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile
+index 56dab563abd7a7..3d09bfbfb30a84 100644
+--- a/drivers/gpu/drm/tests/Makefile
++++ b/drivers/gpu/drm/tests/Makefile
+@@ -4,7 +4,6 @@ obj-$(CONFIG_DRM_KUNIT_TEST_HELPERS) += \
+ drm_kunit_helpers.o
+
+ obj-$(CONFIG_DRM_KUNIT_TEST) += \
+- drm_buddy_test.o \
+ drm_cmdline_parser_test.o \
+ drm_connector_test.o \
+ drm_damage_helper_test.o \
+diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c b/drivers/gpu/drm/tests/drm_buddy_test.c
+deleted file mode 100644
+index 9662c949d0e39c..00000000000000
+--- a/drivers/gpu/drm/tests/drm_buddy_test.c
++++ /dev/null
+@@ -1,778 +0,0 @@
+-// SPDX-License-Identifier: MIT
+-/*
+- * Copyright © 2019 Intel Corporation
+- * Copyright © 2022 MaÃra Canal <[email protected]>
+- */
+-
+-#include <kunit/test.h>
+-
+-#include <linux/prime_numbers.h>
+-#include <linux/sched/signal.h>
+-#include <linux/sizes.h>
+-
+-#include <drm/drm_buddy.h>
+-
+-#include "../lib/drm_random.h"
+-
+-static unsigned int random_seed;
+-
+-static inline u64 get_size(int order, u64 chunk_size)
+-{
+- return (1 << order) * chunk_size;
+-}
+-
+-static void drm_test_buddy_alloc_range_bias(struct kunit *test)
+-{
+- u32 mm_size, size, ps, bias_size, bias_start, bias_end, bias_rem;
+- DRM_RND_STATE(prng, random_seed);
+- unsigned int i, count, *order;
+- struct drm_buddy_block *block;
+- unsigned long flags;
+- struct drm_buddy mm;
+- LIST_HEAD(allocated);
+-
+- bias_size = SZ_1M;
+- ps = roundup_pow_of_two(prandom_u32_state(&prng) % bias_size);
+- ps = max(SZ_4K, ps);
+- mm_size = (SZ_8M-1) & ~(ps-1); /* Multiple roots */
+-
+- kunit_info(test, "mm_size=%u, ps=%u\n", mm_size, ps);
+-
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps),
+- "buddy_init failed\n");
+-
+- count = mm_size / bias_size;
+- order = drm_random_order(count, &prng);
+- KUNIT_EXPECT_TRUE(test, order);
+-
+- /*
+- * Idea is to split the address space into uniform bias ranges, and then
+- * in some random order allocate within each bias, using various
+- * patterns within. This should detect if allocations leak out from a
+- * given bias, for example.
+- */
+-
+- for (i = 0; i < count; i++) {
+- LIST_HEAD(tmp);
+- u32 size;
+-
+- bias_start = order[i] * bias_size;
+- bias_end = bias_start + bias_size;
+- bias_rem = bias_size;
+-
+- /* internal round_up too big */
+- KUNIT_ASSERT_TRUE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, bias_size + ps, bias_size,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, bias_size, bias_size);
+-
+- /* size too big */
+- KUNIT_ASSERT_TRUE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, bias_size + ps, ps,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, bias_size + ps, ps);
+-
+- /* bias range too small for size */
+- KUNIT_ASSERT_TRUE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start + ps,
+- bias_end, bias_size, ps,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start + ps, bias_end, bias_size, ps);
+-
+- /* bias misaligned */
+- KUNIT_ASSERT_TRUE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start + ps,
+- bias_end - ps,
+- bias_size >> 1, bias_size >> 1,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc h didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start + ps, bias_end - ps, bias_size >> 1, bias_size >> 1);
+-
+- /* single big page */
+- KUNIT_ASSERT_FALSE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, bias_size, bias_size,
+- &tmp,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc i failed with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, bias_size, bias_size);
+- drm_buddy_free_list(&mm, &tmp, 0);
+-
+- /* single page with internal round_up */
+- KUNIT_ASSERT_FALSE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, ps, bias_size,
+- &tmp,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, ps, bias_size);
+- drm_buddy_free_list(&mm, &tmp, 0);
+-
+- /* random size within */
+- size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+- if (size)
+- KUNIT_ASSERT_FALSE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, size, ps,
+- &tmp,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, size, ps);
+-
+- bias_rem -= size;
+- /* too big for current avail */
+- KUNIT_ASSERT_TRUE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, bias_rem + ps, ps,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, bias_rem + ps, ps);
+-
+- if (bias_rem) {
+- /* random fill of the remainder */
+- size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+- size = max(size, ps);
+-
+- KUNIT_ASSERT_FALSE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, size, ps,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, size, ps);
+- /*
+- * Intentionally allow some space to be left
+- * unallocated, and ideally not always on the bias
+- * boundaries.
+- */
+- drm_buddy_free_list(&mm, &tmp, 0);
+- } else {
+- list_splice_tail(&tmp, &allocated);
+- }
+- }
+-
+- kfree(order);
+- drm_buddy_free_list(&mm, &allocated, 0);
+- drm_buddy_fini(&mm);
+-
+- /*
+- * Something more free-form. Idea is to pick a random starting bias
+- * range within the address space and then start filling it up. Also
+- * randomly grow the bias range in both directions as we go along. This
+- * should give us bias start/end which is not always uniform like above,
+- * and in some cases will require the allocator to jump over already
+- * allocated nodes in the middle of the address space.
+- */
+-
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps),
+- "buddy_init failed\n");
+-
+- bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
+- bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
+- bias_end = max(bias_end, bias_start + ps);
+- bias_rem = bias_end - bias_start;
+-
+- do {
+- u32 size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+-
+- KUNIT_ASSERT_FALSE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, size, ps,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, size, ps);
+- bias_rem -= size;
+-
+- /*
+- * Try to randomly grow the bias range in both directions, or
+- * only one, or perhaps don't grow at all.
+- */
+- do {
+- u32 old_bias_start = bias_start;
+- u32 old_bias_end = bias_end;
+-
+- if (bias_start)
+- bias_start -= round_up(prandom_u32_state(&prng) % bias_start, ps);
+- if (bias_end != mm_size)
+- bias_end += round_up(prandom_u32_state(&prng) % (mm_size - bias_end), ps);
+-
+- bias_rem += old_bias_start - bias_start;
+- bias_rem += bias_end - old_bias_end;
+- } while (!bias_rem && (bias_start || bias_end != mm_size));
+- } while (bias_rem);
+-
+- KUNIT_ASSERT_EQ(test, bias_start, 0);
+- KUNIT_ASSERT_EQ(test, bias_end, mm_size);
+- KUNIT_ASSERT_TRUE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start, bias_end,
+- ps, ps,
+- &allocated,
+- DRM_BUDDY_RANGE_ALLOCATION),
+- "buddy_alloc passed with bias(%x-%x), size=%u\n",
+- bias_start, bias_end, ps);
+-
+- drm_buddy_free_list(&mm, &allocated, 0);
+- drm_buddy_fini(&mm);
+-
+- /*
+- * Allocate cleared blocks in the bias range when the DRM buddy's clear avail is
+- * zero. This will validate the bias range allocation in scenarios like system boot
+- * when no cleared blocks are available and exercise the fallback path too. The resulting
+- * blocks should always be dirty.
+- */
+-
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps),
+- "buddy_init failed\n");
+-
+- bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
+- bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
+- bias_end = max(bias_end, bias_start + ps);
+- bias_rem = bias_end - bias_start;
+-
+- flags = DRM_BUDDY_CLEAR_ALLOCATION | DRM_BUDDY_RANGE_ALLOCATION;
+- size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+-
+- KUNIT_ASSERT_FALSE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, bias_start,
+- bias_end, size, ps,
+- &allocated,
+- flags),
+- "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+- bias_start, bias_end, size, ps);
+-
+- list_for_each_entry(block, &allocated, link)
+- KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
+-
+- drm_buddy_free_list(&mm, &allocated, 0);
+- drm_buddy_fini(&mm);
+-}
+-
+-static void drm_test_buddy_alloc_clear(struct kunit *test)
+-{
+- unsigned long n_pages, total, i = 0;
+- DRM_RND_STATE(prng, random_seed);
+- const unsigned long ps = SZ_4K;
+- struct drm_buddy_block *block;
+- const int max_order = 12;
+- LIST_HEAD(allocated);
+- struct drm_buddy mm;
+- unsigned int order;
+- u32 mm_size, size;
+- LIST_HEAD(dirty);
+- LIST_HEAD(clean);
+-
+- mm_size = SZ_4K << max_order;
+- KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
+-
+- KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+-
+- /*
+- * Idea is to allocate and free some random portion of the address space,
+- * returning those pages as non-dirty and randomly alternate between
+- * requesting dirty and non-dirty pages (not going over the limit
+- * we freed as non-dirty), putting that into two separate lists.
+- * Loop over both lists at the end checking that the dirty list
+- * is indeed all dirty pages and vice versa. Free it all again,
+- * keeping the dirty/clear status.
+- */
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 5 * ps, ps, &allocated,
+- DRM_BUDDY_TOPDOWN_ALLOCATION),
+- "buddy_alloc hit an error size=%lu\n", 5 * ps);
+- drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
+-
+- n_pages = 10;
+- do {
+- unsigned long flags;
+- struct list_head *list;
+- int slot = i % 2;
+-
+- if (slot == 0) {
+- list = &dirty;
+- flags = 0;
+- } else {
+- list = &clean;
+- flags = DRM_BUDDY_CLEAR_ALLOCATION;
+- }
+-
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- ps, ps, list,
+- flags),
+- "buddy_alloc hit an error size=%lu\n", ps);
+- } while (++i < n_pages);
+-
+- list_for_each_entry(block, &clean, link)
+- KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), true);
+-
+- list_for_each_entry(block, &dirty, link)
+- KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
+-
+- drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
+-
+- /*
+- * Trying to go over the clear limit for some allocation.
+- * The allocation should never fail with reasonable page-size.
+- */
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 10 * ps, ps, &clean,
+- DRM_BUDDY_CLEAR_ALLOCATION),
+- "buddy_alloc hit an error size=%lu\n", 10 * ps);
+-
+- drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
+- drm_buddy_free_list(&mm, &dirty, 0);
+- drm_buddy_fini(&mm);
+-
+- KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
+-
+- /*
+- * Create a new mm. Intentionally fragment the address space by creating
+- * two alternating lists. Free both lists, one as dirty the other as clean.
+- * Try to allocate double the previous size with matching min_page_size. The
+- * allocation should never fail as it calls the force_merge. Also check that
+- * the page is always dirty after force_merge. Free the page as dirty, then
+- * repeat the whole thing, increment the order until we hit the max_order.
+- */
+-
+- i = 0;
+- n_pages = mm_size / ps;
+- do {
+- struct list_head *list;
+- int slot = i % 2;
+-
+- if (slot == 0)
+- list = &dirty;
+- else
+- list = &clean;
+-
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- ps, ps, list, 0),
+- "buddy_alloc hit an error size=%lu\n", ps);
+- } while (++i < n_pages);
+-
+- drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
+- drm_buddy_free_list(&mm, &dirty, 0);
+-
+- order = 1;
+- do {
+- size = SZ_4K << order;
+-
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- size, size, &allocated,
+- DRM_BUDDY_CLEAR_ALLOCATION),
+- "buddy_alloc hit an error size=%u\n", size);
+- total = 0;
+- list_for_each_entry(block, &allocated, link) {
+- if (size != mm_size)
+- KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
+- total += drm_buddy_block_size(&mm, block);
+- }
+- KUNIT_EXPECT_EQ(test, total, size);
+-
+- drm_buddy_free_list(&mm, &allocated, 0);
+- } while (++order <= max_order);
+-
+- drm_buddy_fini(&mm);
+-
+- /*
+- * Create a new mm with a non power-of-two size. Allocate a random size, free as
+- * cleared and then call fini. This will ensure the multi-root force merge during
+- * fini.
+- */
+- mm_size = 12 * SZ_4K;
+- size = max(round_up(prandom_u32_state(&prng) % mm_size, ps), ps);
+- KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- size, ps, &allocated,
+- DRM_BUDDY_TOPDOWN_ALLOCATION),
+- "buddy_alloc hit an error size=%u\n", size);
+- drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
+- drm_buddy_fini(&mm);
+-}
+-
+-static void drm_test_buddy_alloc_contiguous(struct kunit *test)
+-{
+- const unsigned long ps = SZ_4K, mm_size = 16 * 3 * SZ_4K;
+- unsigned long i, n_pages, total;
+- struct drm_buddy_block *block;
+- struct drm_buddy mm;
+- LIST_HEAD(left);
+- LIST_HEAD(middle);
+- LIST_HEAD(right);
+- LIST_HEAD(allocated);
+-
+- KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
+-
+- /*
+- * Idea is to fragment the address space by alternating block
+- * allocations between three different lists; one for left, middle and
+- * right. We can then free a list to simulate fragmentation. In
+- * particular we want to exercise the DRM_BUDDY_CONTIGUOUS_ALLOCATION,
+- * including the try_harder path.
+- */
+-
+- i = 0;
+- n_pages = mm_size / ps;
+- do {
+- struct list_head *list;
+- int slot = i % 3;
+-
+- if (slot == 0)
+- list = &left;
+- else if (slot == 1)
+- list = &middle;
+- else
+- list = &right;
+- KUNIT_ASSERT_FALSE_MSG(test,
+- drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- ps, ps, list, 0),
+- "buddy_alloc hit an error size=%lu\n",
+- ps);
+- } while (++i < n_pages);
+-
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 3 * ps, ps, &allocated,
+- DRM_BUDDY_CONTIGUOUS_ALLOCATION),
+- "buddy_alloc didn't error size=%lu\n", 3 * ps);
+-
+- drm_buddy_free_list(&mm, &middle, 0);
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 3 * ps, ps, &allocated,
+- DRM_BUDDY_CONTIGUOUS_ALLOCATION),
+- "buddy_alloc didn't error size=%lu\n", 3 * ps);
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 2 * ps, ps, &allocated,
+- DRM_BUDDY_CONTIGUOUS_ALLOCATION),
+- "buddy_alloc didn't error size=%lu\n", 2 * ps);
+-
+- drm_buddy_free_list(&mm, &right, 0);
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 3 * ps, ps, &allocated,
+- DRM_BUDDY_CONTIGUOUS_ALLOCATION),
+- "buddy_alloc didn't error size=%lu\n", 3 * ps);
+- /*
+- * At this point we should have enough contiguous space for 2 blocks,
+- * however they are never buddies (since we freed middle and right) so
+- * will require the try_harder logic to find them.
+- */
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 2 * ps, ps, &allocated,
+- DRM_BUDDY_CONTIGUOUS_ALLOCATION),
+- "buddy_alloc hit an error size=%lu\n", 2 * ps);
+-
+- drm_buddy_free_list(&mm, &left, 0);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+- 3 * ps, ps, &allocated,
+- DRM_BUDDY_CONTIGUOUS_ALLOCATION),
+- "buddy_alloc hit an error size=%lu\n", 3 * ps);
+-
+- total = 0;
+- list_for_each_entry(block, &allocated, link)
+- total += drm_buddy_block_size(&mm, block);
+-
+- KUNIT_ASSERT_EQ(test, total, ps * 2 + ps * 3);
+-
+- drm_buddy_free_list(&mm, &allocated, 0);
+- drm_buddy_fini(&mm);
+-}
+-
+-static void drm_test_buddy_alloc_pathological(struct kunit *test)
+-{
+- u64 mm_size, size, start = 0;
+- struct drm_buddy_block *block;
+- const int max_order = 3;
+- unsigned long flags = 0;
+- int order, top;
+- struct drm_buddy mm;
+- LIST_HEAD(blocks);
+- LIST_HEAD(holes);
+- LIST_HEAD(tmp);
+-
+- /*
+- * Create a pot-sized mm, then allocate one of each possible
+- * order within. This should leave the mm with exactly one
+- * page left. Free the largest block, then whittle down again.
+- * Eventually we will have a fully 50% fragmented mm.
+- */
+-
+- mm_size = SZ_4K << max_order;
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
+- "buddy_init failed\n");
+-
+- KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+-
+- for (top = max_order; top; top--) {
+- /* Make room by freeing the largest allocated block */
+- block = list_first_entry_or_null(&blocks, typeof(*block), link);
+- if (block) {
+- list_del(&block->link);
+- drm_buddy_free_block(&mm, block);
+- }
+-
+- for (order = top; order--;) {
+- size = get_size(order, mm.chunk_size);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start,
+- mm_size, size, size,
+- &tmp, flags),
+- "buddy_alloc hit -ENOMEM with order=%d, top=%d\n",
+- order, top);
+-
+- block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+- KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+-
+- list_move_tail(&block->link, &blocks);
+- }
+-
+- /* There should be one final page for this sub-allocation */
+- size = get_size(0, mm.chunk_size);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc hit -ENOMEM for hole\n");
+-
+- block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+- KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+-
+- list_move_tail(&block->link, &holes);
+-
+- size = get_size(top, mm.chunk_size);
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!",
+- top, max_order);
+- }
+-
+- drm_buddy_free_list(&mm, &holes, 0);
+-
+- /* Nothing larger than blocks of chunk_size now available */
+- for (order = 1; order <= max_order; order++) {
+- size = get_size(order, mm.chunk_size);
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc unexpectedly succeeded at order %d, it should be full!",
+- order);
+- }
+-
+- list_splice_tail(&holes, &blocks);
+- drm_buddy_free_list(&mm, &blocks, 0);
+- drm_buddy_fini(&mm);
+-}
+-
+-static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
+-{
+- u64 mm_size, size, start = 0;
+- struct drm_buddy_block *block, *bn;
+- const unsigned int max_order = 16;
+- unsigned long flags = 0;
+- struct drm_buddy mm;
+- unsigned int order;
+- LIST_HEAD(blocks);
+- LIST_HEAD(tmp);
+-
+- /*
+- * Create a pot-sized mm, then allocate one of each possible
+- * order within. This should leave the mm with exactly one
+- * page left.
+- */
+-
+- mm_size = SZ_4K << max_order;
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
+- "buddy_init failed\n");
+-
+- KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+-
+- for (order = 0; order < max_order; order++) {
+- size = get_size(order, mm.chunk_size);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc hit -ENOMEM with order=%d\n",
+- order);
+-
+- block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+- KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+-
+- list_move_tail(&block->link, &blocks);
+- }
+-
+- /* And now the last remaining block available */
+- size = get_size(0, mm.chunk_size);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc hit -ENOMEM on final alloc\n");
+-
+- block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+- KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+-
+- list_move_tail(&block->link, &blocks);
+-
+- /* Should be completely full! */
+- for (order = max_order; order--;) {
+- size = get_size(order, mm.chunk_size);
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc unexpectedly succeeded, it should be full!");
+- }
+-
+- block = list_last_entry(&blocks, typeof(*block), link);
+- list_del(&block->link);
+- drm_buddy_free_block(&mm, block);
+-
+- /* As we free in increasing size, we make available larger blocks */
+- order = 1;
+- list_for_each_entry_safe(block, bn, &blocks, link) {
+- list_del(&block->link);
+- drm_buddy_free_block(&mm, block);
+-
+- size = get_size(order, mm.chunk_size);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc hit -ENOMEM with order=%d\n",
+- order);
+-
+- block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+- KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+-
+- list_del(&block->link);
+- drm_buddy_free_block(&mm, block);
+- order++;
+- }
+-
+- /* To confirm, now the whole mm should be available */
+- size = get_size(max_order, mm.chunk_size);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
+- max_order);
+-
+- block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+- KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+-
+- list_del(&block->link);
+- drm_buddy_free_block(&mm, block);
+- drm_buddy_free_list(&mm, &blocks, 0);
+- drm_buddy_fini(&mm);
+-}
+-
+-static void drm_test_buddy_alloc_optimistic(struct kunit *test)
+-{
+- u64 mm_size, size, start = 0;
+- struct drm_buddy_block *block;
+- unsigned long flags = 0;
+- const int max_order = 16;
+- struct drm_buddy mm;
+- LIST_HEAD(blocks);
+- LIST_HEAD(tmp);
+- int order;
+-
+- /*
+- * Create a mm with one block of each order available, and
+- * try to allocate them all.
+- */
+-
+- mm_size = SZ_4K * ((1 << (max_order + 1)) - 1);
+-
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
+- "buddy_init failed\n");
+-
+- KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+-
+- for (order = 0; order <= max_order; order++) {
+- size = get_size(order, mm.chunk_size);
+- KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc hit -ENOMEM with order=%d\n",
+- order);
+-
+- block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+- KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+-
+- list_move_tail(&block->link, &blocks);
+- }
+-
+- /* Should be completely full! */
+- size = get_size(0, mm.chunk_size);
+- KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+- size, size, &tmp, flags),
+- "buddy_alloc unexpectedly succeeded, it should be full!");
+-
+- drm_buddy_free_list(&mm, &blocks, 0);
+- drm_buddy_fini(&mm);
+-}
+-
+-static void drm_test_buddy_alloc_limit(struct kunit *test)
+-{
+- u64 size = U64_MAX, start = 0;
+- struct drm_buddy_block *block;
+- unsigned long flags = 0;
+- LIST_HEAD(allocated);
+- struct drm_buddy mm;
+-
+- KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, size, SZ_4K));
+-
+- KUNIT_EXPECT_EQ_MSG(test, mm.max_order, DRM_BUDDY_MAX_ORDER,
+- "mm.max_order(%d) != %d\n", mm.max_order,
+- DRM_BUDDY_MAX_ORDER);
+-
+- size = mm.chunk_size << mm.max_order;
+- KUNIT_EXPECT_FALSE(test, drm_buddy_alloc_blocks(&mm, start, size, size,
+- mm.chunk_size, &allocated, flags));
+-
+- block = list_first_entry_or_null(&allocated, struct drm_buddy_block, link);
+- KUNIT_EXPECT_TRUE(test, block);
+-
+- KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_order(block), mm.max_order,
+- "block order(%d) != %d\n",
+- drm_buddy_block_order(block), mm.max_order);
+-
+- KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_size(&mm, block),
+- BIT_ULL(mm.max_order) * mm.chunk_size,
+- "block size(%llu) != %llu\n",
+- drm_buddy_block_size(&mm, block),
+- BIT_ULL(mm.max_order) * mm.chunk_size);
+-
+- drm_buddy_free_list(&mm, &allocated, 0);
+- drm_buddy_fini(&mm);
+-}
+-
+-static int drm_buddy_suite_init(struct kunit_suite *suite)
+-{
+- while (!random_seed)
+- random_seed = get_random_u32();
+-
+- kunit_info(suite, "Testing DRM buddy manager, with random_seed=0x%x\n",
+- random_seed);
+-
+- return 0;
+-}
+-
+-static struct kunit_case drm_buddy_tests[] = {
+- KUNIT_CASE(drm_test_buddy_alloc_limit),
+- KUNIT_CASE(drm_test_buddy_alloc_optimistic),
+- KUNIT_CASE(drm_test_buddy_alloc_pessimistic),
+- KUNIT_CASE(drm_test_buddy_alloc_pathological),
+- KUNIT_CASE(drm_test_buddy_alloc_contiguous),
+- KUNIT_CASE(drm_test_buddy_alloc_clear),
+- KUNIT_CASE(drm_test_buddy_alloc_range_bias),
+- {}
+-};
+-
+-static struct kunit_suite drm_buddy_test_suite = {
+- .name = "drm_buddy",
+- .suite_init = drm_buddy_suite_init,
+- .test_cases = drm_buddy_tests,
+-};
+-
+-kunit_test_suite(drm_buddy_test_suite);
+-
+-MODULE_AUTHOR("Intel Corporation");
+-MODULE_DESCRIPTION("Kunit test for drm_buddy functions");
+-MODULE_LICENSE("GPL");
+diff --git a/drivers/gpu/drm/tests/drm_exec_test.c b/drivers/gpu/drm/tests/drm_exec_test.c
+index d6c4dd1194a0eb..c1f7a747752fea 100644
+--- a/drivers/gpu/drm/tests/drm_exec_test.c
++++ b/drivers/gpu/drm/tests/drm_exec_test.c
+@@ -16,8 +16,6 @@
+ #include <drm/drm_gem.h>
+ #include <drm/drm_kunit_helpers.h>
+
+-#include "../lib/drm_random.h"
+-
+ struct drm_exec_priv {
+ struct device *dev;
+ struct drm_device *drm;
+diff --git a/drivers/gpu/drm/tests/drm_gem_shmem_test.c b/drivers/gpu/drm/tests/drm_gem_shmem_test.c
+index 0c8c46ca158c06..cc40d63ab809e7 100644
+--- a/drivers/gpu/drm/tests/drm_gem_shmem_test.c
++++ b/drivers/gpu/drm/tests/drm_gem_shmem_test.c
+@@ -90,13 +90,9 @@ static void drm_gem_shmem_test_obj_create_private(struct kunit *test)
+ sg_init_one(sgt->sgl, buf, TEST_SIZE);
+
+ /*
+- * Set the DMA mask to 64-bits and map the sgtables
+- * otherwise drm_gem_shmem_free will cause a warning
+- * on debug kernels.
++ * Map the sgtables otherwise drm_gem_shmem_free will cause a warning on
++ * debug kernels.
+ */
+- ret = dma_set_mask(drm_dev->dev, DMA_BIT_MASK(64));
+- KUNIT_ASSERT_EQ(test, ret, 0);
+-
+ ret = dma_map_sgtable(drm_dev->dev, sgt, DMA_BIDIRECTIONAL, 0);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+@@ -342,11 +338,19 @@ static int drm_gem_shmem_test_init(struct kunit *test)
+ {
+ struct device *dev;
+ struct drm_device *drm_dev;
++ int ret;
+
+ /* Allocate a parent device */
+ dev = drm_kunit_helper_alloc_device(test);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
++ /*
++ * Set the DMA mask to 64-bits to avoid intermittent failures calling
++ * drm_gem_shmem_get_pages_sgt().
++ */
++ ret = dma_set_mask(dev, DMA_BIT_MASK(64));
++ KUNIT_ASSERT_EQ(test, ret, 0);
++
+ /*
+ * The DRM core will automatically initialize the GEM core and create
+ * a DRM Memory Manager object which provides an address space pool
+diff --git a/drivers/gpu/drm/tests/drm_mm_test.c b/drivers/gpu/drm/tests/drm_mm_test.c
+index 6174d092902082..99744322ed1a25 100644
+--- a/drivers/gpu/drm/tests/drm_mm_test.c
++++ b/drivers/gpu/drm/tests/drm_mm_test.c
+@@ -15,8 +15,6 @@
+
+ #include <drm/drm_mm.h>
+
+-#include "../lib/drm_random.h"
+-
+ enum {
+ BEST,
+ BOTTOMUP,
+diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
+index 1adf18481ea057..7b86da22a1a5cb 100644
+--- a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
++++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
+@@ -251,7 +251,7 @@ static void ttm_bo_validate_basic(struct kunit *test)
+ NULL, &dummy_ttm_bo_destroy);
+ KUNIT_EXPECT_EQ(test, err, 0);
+
+- snd_place = ttm_place_kunit_init(test, snd_mem, DRM_BUDDY_TOPDOWN_ALLOCATION);
++ snd_place = ttm_place_kunit_init(test, snd_mem, GPU_BUDDY_TOPDOWN_ALLOCATION);
+ snd_placement = ttm_placement_kunit_init(test, snd_place, 1);
+
+ err = ttm_bo_validate(bo, snd_placement, &ctx_val);
+@@ -263,7 +263,7 @@ static void ttm_bo_validate_basic(struct kunit *test)
+ KUNIT_EXPECT_TRUE(test, ttm_tt_is_populated(bo->ttm));
+ KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem);
+ KUNIT_EXPECT_EQ(test, bo->resource->placement,
+- DRM_BUDDY_TOPDOWN_ALLOCATION);
++ GPU_BUDDY_TOPDOWN_ALLOCATION);
+
+ ttm_bo_put(bo);
+ ttm_mock_manager_fini(priv->ttm_dev, snd_mem);
+diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c
+index f6d1c8a2845dd5..8b2d979a87077c 100644
+--- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c
++++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c
+@@ -27,7 +27,7 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man,
+ {
+ struct ttm_mock_manager *manager = to_mock_mgr(man);
+ struct ttm_mock_resource *mock_res;
+- struct drm_buddy *mm = &manager->mm;
++ struct gpu_buddy *mm = &manager->mm;
+ u64 lpfn, fpfn, alloc_size;
+ int err;
+
+@@ -43,14 +43,14 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man,
+ INIT_LIST_HEAD(&mock_res->blocks);
+
+ if (place->flags & TTM_PL_FLAG_TOPDOWN)
+- mock_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
++ mock_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
+
+ if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
+- mock_res->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
++ mock_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
+
+ alloc_size = (uint64_t)mock_res->base.size;
+ mutex_lock(&manager->lock);
+- err = drm_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size,
++ err = gpu_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size,
+ manager->default_page_size,
+ &mock_res->blocks,
+ mock_res->flags);
+@@ -63,7 +63,7 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man,
+ return 0;
+
+ error_free_blocks:
+- drm_buddy_free_list(mm, &mock_res->blocks, 0);
++ gpu_buddy_free_list(mm, &mock_res->blocks, 0);
+ ttm_resource_fini(man, &mock_res->base);
+ mutex_unlock(&manager->lock);
+
+@@ -75,10 +75,10 @@ static void ttm_mock_manager_free(struct ttm_resource_manager *man,
+ {
+ struct ttm_mock_manager *manager = to_mock_mgr(man);
+ struct ttm_mock_resource *mock_res = to_mock_mgr_resource(res);
+- struct drm_buddy *mm = &manager->mm;
++ struct gpu_buddy *mm = &manager->mm;
+
+ mutex_lock(&manager->lock);
+- drm_buddy_free_list(mm, &mock_res->blocks, 0);
++ gpu_buddy_free_list(mm, &mock_res->blocks, 0);
+ mutex_unlock(&manager->lock);
+
+ ttm_resource_fini(man, res);
+@@ -102,7 +102,7 @@ int ttm_mock_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size)
+
+ mutex_init(&manager->lock);
+
+- err = drm_buddy_init(&manager->mm, size, PAGE_SIZE);
++ err = gpu_buddy_init(&manager->mm, size, PAGE_SIZE);
+
+ if (err) {
+ kfree(manager);
+@@ -138,7 +138,7 @@ void ttm_mock_manager_fini(struct ttm_device *bdev, u32 mem_type)
+ ttm_resource_manager_set_used(man, false);
+
+ mutex_lock(&mock_man->lock);
+- drm_buddy_fini(&mock_man->mm);
++ gpu_buddy_fini(&mock_man->mm);
+ mutex_unlock(&mock_man->lock);
+
+ ttm_set_driver_manager(bdev, mem_type, NULL);
+diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h
+index e4c95f86a4670a..08710756fd8e0f 100644
+--- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h
++++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h
+@@ -5,11 +5,11 @@
+ #ifndef TTM_MOCK_MANAGER_H
+ #define TTM_MOCK_MANAGER_H
+
+-#include <drm/drm_buddy.h>
++#include <linux/gpu_buddy.h>
+
+ struct ttm_mock_manager {
+ struct ttm_resource_manager man;
+- struct drm_buddy mm;
++ struct gpu_buddy mm;
+ u64 default_page_size;
+ /* protects allocations of mock buffer objects */
+ struct mutex lock;
+diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
+index 3740e0552960d0..e49a4950b416da 100644
+--- a/drivers/gpu/drm/vc4/vc4_bo.c
++++ b/drivers/gpu/drm/vc4/vc4_bo.c
+@@ -733,9 +733,13 @@ static int vc4_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct
+ {
+ struct vc4_bo *bo = to_vc4_bo(obj);
+
+- if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) {
+- DRM_DEBUG("mmapping of shader BOs for writing not allowed.\n");
+- return -EINVAL;
++ if (bo->validated_shader) {
++ if (vma->vm_flags & VM_WRITE) {
++ DRM_DEBUG("mmapping of shader BOs for writing not allowed.\n");
++ return -EINVAL;
++ }
++
++ vm_flags_clear(vma, VM_MAYWRITE);
+ }
+
+ mutex_lock(&bo->madv_lock);
+diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
+index 7dfb2006c561ca..ec31a41d33d678 100644
+--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
++++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
+@@ -48,7 +48,10 @@ static void virtio_gpu_config_changed_work_func(struct work_struct *work)
+ virtio_gpu_cmd_get_edids(vgdev);
+ virtio_gpu_cmd_get_display_info(vgdev);
+ virtio_gpu_notify(vgdev);
+- drm_helper_hpd_irq_event(vgdev->ddev);
++ wait_event_timeout(vgdev->resp_wq,
++ !vgdev->display_info_pending,
++ 5 * HZ);
++ drm_kms_helper_hotplug_event(vgdev->ddev);
+ }
+ events_clear |= VIRTIO_GPU_EVENT_DISPLAY;
+ }
+diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
+index 0d3d0d09f39b9f..4369275cb56b5b 100644
+--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
++++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
+@@ -669,9 +669,6 @@ static void virtio_gpu_cmd_get_display_info_cb(struct virtio_gpu_device *vgdev,
+ vgdev->display_info_pending = false;
+ spin_unlock(&vgdev->display_info_lock);
+ wake_up(&vgdev->resp_wq);
+-
+- if (!drm_helper_hpd_irq_event(vgdev->ddev))
+- drm_kms_helper_hotplug_event(vgdev->ddev);
+ }
+
+ static void virtio_gpu_cmd_get_capset_info_cb(struct virtio_gpu_device *vgdev,
+@@ -726,7 +723,8 @@ static int virtio_get_edid_block(void *data, u8 *buf,
+ struct virtio_gpu_resp_edid *resp = data;
+ size_t start = block * EDID_LENGTH;
+
+- if (start + len > le32_to_cpu(resp->size))
++ if (start + len > le32_to_cpu(resp->size) ||
++ start + len > sizeof(resp->edid))
+ return -EINVAL;
+ memcpy(buf, resp->edid + start, len);
+ return 0;
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+index d7a8070330ba54..06d80fad4ceea9 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+@@ -96,7 +96,7 @@ static int vmw_gb_surface_unbind(struct vmw_resource *res,
+ static int vmw_gb_surface_destroy(struct vmw_resource *res);
+ static int
+ vmw_gb_surface_define_internal(struct drm_device *dev,
+- struct drm_vmw_gb_surface_create_ext_req *req,
++ const struct drm_vmw_gb_surface_create_ext_req *req,
+ struct drm_vmw_gb_surface_create_rep *rep,
+ struct drm_file *file_priv);
+ static int
+@@ -1538,7 +1538,7 @@ int vmw_gb_surface_reference_ext_ioctl(struct drm_device *dev, void *data,
+ */
+ static int
+ vmw_gb_surface_define_internal(struct drm_device *dev,
+- struct drm_vmw_gb_surface_create_ext_req *req,
++ const struct drm_vmw_gb_surface_create_ext_req *req,
+ struct drm_vmw_gb_surface_create_rep *rep,
+ struct drm_file *file_priv)
+ {
+@@ -1556,9 +1556,21 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
+ req->base.svga3d_flags);
+
+ /* array_size must be null for non-GL3 host. */
+- if (req->base.array_size > 0 && !has_sm4_context(dev_priv)) {
+- VMW_DEBUG_USER("SM4 surface not supported.\n");
+- return -EINVAL;
++ if (req->base.array_size > 0) {
++ if (has_sm5_context(dev_priv)) {
++ if (req->base.array_size > SVGA3D_SM5_MAX_SURFACE_ARRAYSIZE) {
++ VMW_DEBUG_USER("Invalid Surface Array Size.\n");
++ return -EINVAL;
++ }
++ } else if (has_sm4_context(dev_priv)) {
++ if (req->base.array_size > SVGA3D_SM4_MAX_SURFACE_ARRAYSIZE) {
++ VMW_DEBUG_USER("Invalid Surface Array Size.\n");
++ return -EINVAL;
++ }
++ } else {
++ VMW_DEBUG_USER("SM4+ surface not supported.\n");
++ return -EINVAL;
++ }
+ }
+
+ if (!has_sm4_1_context(dev_priv)) {
+diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
+index 422dc351dc0e05..0025871485cb43 100644
+--- a/drivers/gpu/drm/xe/xe_pt.c
++++ b/drivers/gpu/drm/xe/xe_pt.c
+@@ -868,12 +868,22 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *t
+ u64 *ptr = data;
+ u32 i;
+
++ /*
++ * @qword_ofs is the absolute entry offset within the page table, while
++ * @ptes is indexed relative to @update->ofs (its first entry). The GPU
++ * path (write_pgtable) splits a single update into MAX_PTE_PER_SDI-sized
++ * chunks, calling this with an advancing @qword_ofs but a fresh @data
++ * pointer per chunk, so translate back into a @ptes index rather than
++ * assuming the chunk starts at ptes[0].
++ */
+ for (i = 0; i < num_qwords; i++) {
++ u32 idx = qword_ofs - update->ofs + i;
++
+ if (map)
+ xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
+- sizeof(u64), u64, ptes[i].pte);
++ sizeof(u64), u64, ptes[idx].pte);
+ else
+- ptr[i] = ptes[i].pte;
++ ptr[i] = ptes[idx].pte;
+ }
+ }
+
+diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h
+index dca374b6521c2d..cbc76747021e59 100644
+--- a/drivers/gpu/drm/xe/xe_res_cursor.h
++++ b/drivers/gpu/drm/xe/xe_res_cursor.h
+@@ -44,10 +44,10 @@ struct xe_res_cursor {
+ void *node;
+ u32 mem_type;
+ struct scatterlist *sgl;
+- struct drm_buddy *mm;
++ struct gpu_buddy *mm;
+ };
+
+-static struct drm_buddy *xe_res_get_buddy(struct ttm_resource *res)
++static struct gpu_buddy *xe_res_get_buddy(struct ttm_resource *res)
+ {
+ struct ttm_resource_manager *mgr;
+
+@@ -81,30 +81,30 @@ static inline void xe_res_first(struct ttm_resource *res,
+ case XE_PL_STOLEN:
+ case XE_PL_VRAM0:
+ case XE_PL_VRAM1: {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ struct list_head *head, *next;
+- struct drm_buddy *mm = xe_res_get_buddy(res);
++ struct gpu_buddy *mm = xe_res_get_buddy(res);
+
+ head = &to_xe_ttm_vram_mgr_resource(res)->blocks;
+
+ block = list_first_entry_or_null(head,
+- struct drm_buddy_block,
++ struct gpu_buddy_block,
+ link);
+ if (!block)
+ goto fallback;
+
+- while (start >= drm_buddy_block_size(mm, block)) {
+- start -= drm_buddy_block_size(mm, block);
++ while (start >= gpu_buddy_block_size(mm, block)) {
++ start -= gpu_buddy_block_size(mm, block);
+
+ next = block->link.next;
+ if (next != head)
+- block = list_entry(next, struct drm_buddy_block,
++ block = list_entry(next, struct gpu_buddy_block,
+ link);
+ }
+
+ cur->mm = mm;
+- cur->start = drm_buddy_block_offset(block) + start;
+- cur->size = min(drm_buddy_block_size(mm, block) - start,
++ cur->start = gpu_buddy_block_offset(block) + start;
++ cur->size = min(gpu_buddy_block_size(mm, block) - start,
+ size);
+ cur->remaining = size;
+ cur->node = block;
+@@ -175,7 +175,7 @@ static inline void xe_res_first_sg(const struct sg_table *sg,
+ */
+ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)
+ {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ struct list_head *next;
+ u64 start;
+
+@@ -205,18 +205,18 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)
+ block = cur->node;
+
+ next = block->link.next;
+- block = list_entry(next, struct drm_buddy_block, link);
++ block = list_entry(next, struct gpu_buddy_block, link);
+
+
+- while (start >= drm_buddy_block_size(cur->mm, block)) {
+- start -= drm_buddy_block_size(cur->mm, block);
++ while (start >= gpu_buddy_block_size(cur->mm, block)) {
++ start -= gpu_buddy_block_size(cur->mm, block);
+
+ next = block->link.next;
+- block = list_entry(next, struct drm_buddy_block, link);
++ block = list_entry(next, struct gpu_buddy_block, link);
+ }
+
+- cur->start = drm_buddy_block_offset(block) + start;
+- cur->size = min(drm_buddy_block_size(cur->mm, block) - start,
++ cur->start = gpu_buddy_block_offset(block) + start;
++ cur->size = min(gpu_buddy_block_size(cur->mm, block) - start,
+ cur->remaining);
+ cur->node = block;
+ break;
+diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+index 423b261ea74305..8a9f135e368a3b 100644
+--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
++++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+@@ -5,6 +5,7 @@
+ */
+
+ #include <drm/drm_managed.h>
++#include <drm/drm_buddy.h>
+
+ #include <drm/ttm/ttm_placement.h>
+ #include <drm/ttm/ttm_range_manager.h>
+@@ -15,16 +16,16 @@
+ #include "xe_res_cursor.h"
+ #include "xe_ttm_vram_mgr.h"
+
+-static inline struct drm_buddy_block *
++static inline struct gpu_buddy_block *
+ xe_ttm_vram_mgr_first_block(struct list_head *list)
+ {
+- return list_first_entry_or_null(list, struct drm_buddy_block, link);
++ return list_first_entry_or_null(list, struct gpu_buddy_block, link);
+ }
+
+-static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,
++static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm,
+ struct list_head *head)
+ {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+ u64 start, size;
+
+ block = xe_ttm_vram_mgr_first_block(head);
+@@ -32,12 +33,12 @@ static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,
+ return false;
+
+ while (head != block->link.next) {
+- start = drm_buddy_block_offset(block);
+- size = drm_buddy_block_size(mm, block);
++ start = gpu_buddy_block_offset(block);
++ size = gpu_buddy_block_size(mm, block);
+
+- block = list_entry(block->link.next, struct drm_buddy_block,
++ block = list_entry(block->link.next, struct gpu_buddy_block,
+ link);
+- if (start + size != drm_buddy_block_offset(block))
++ if (start + size != gpu_buddy_block_offset(block))
+ return false;
+ }
+
+@@ -51,7 +52,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
+ {
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
+ struct xe_ttm_vram_mgr_resource *vres;
+- struct drm_buddy *mm = &mgr->mm;
++ struct gpu_buddy *mm = &mgr->mm;
+ u64 size, remaining_size, min_page_size;
+ unsigned long lpfn;
+ int err;
+@@ -78,10 +79,10 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
+ INIT_LIST_HEAD(&vres->blocks);
+
+ if (place->flags & TTM_PL_FLAG_TOPDOWN)
+- vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
++ vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
+
+ if (place->fpfn || lpfn != man->size >> PAGE_SHIFT)
+- vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
++ vres->flags |= GPU_BUDDY_RANGE_ALLOCATION;
+
+ if (WARN_ON(!vres->base.size)) {
+ err = -EINVAL;
+@@ -137,7 +138,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
+ */
+ u64 alloc_size = min_t(u64, remaining_size, SZ_2G);
+
+- err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
++ err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
+ (u64)lpfn << PAGE_SHIFT,
+ alloc_size,
+ min_page_size,
+@@ -150,20 +151,20 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
+ } while (remaining_size);
+
+ if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
+- if (!drm_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks))
++ if (!gpu_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks))
+ size = vres->base.size;
+ }
+
+ if (lpfn <= mgr->visible_size >> PAGE_SHIFT) {
+ vres->used_visible_size = size;
+ } else {
+- struct drm_buddy_block *block;
++ struct gpu_buddy_block *block;
+
+ list_for_each_entry(block, &vres->blocks, link) {
+- u64 start = drm_buddy_block_offset(block);
++ u64 start = gpu_buddy_block_offset(block);
+
+ if (start < mgr->visible_size) {
+- u64 end = start + drm_buddy_block_size(mm, block);
++ u64 end = start + gpu_buddy_block_size(mm, block);
+
+ vres->used_visible_size +=
+ min(end, mgr->visible_size) - start;
+@@ -183,11 +184,11 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
+ * the object.
+ */
+ if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) {
+- struct drm_buddy_block *block = list_first_entry(&vres->blocks,
++ struct gpu_buddy_block *block = list_first_entry(&vres->blocks,
+ typeof(*block),
+ link);
+
+- vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT;
++ vres->base.start = gpu_buddy_block_offset(block) >> PAGE_SHIFT;
+ } else {
+ vres->base.start = XE_BO_INVALID_OFFSET;
+ }
+@@ -196,7 +197,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
+ return 0;
+
+ error_free_blocks:
+- drm_buddy_free_list(mm, &vres->blocks, 0);
++ gpu_buddy_free_list(mm, &vres->blocks, 0);
+ mutex_unlock(&mgr->lock);
+ error_fini:
+ ttm_resource_fini(man, &vres->base);
+@@ -211,10 +212,10 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
+ struct xe_ttm_vram_mgr_resource *vres =
+ to_xe_ttm_vram_mgr_resource(res);
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
+- struct drm_buddy *mm = &mgr->mm;
++ struct gpu_buddy *mm = &mgr->mm;
+
+ mutex_lock(&mgr->lock);
+- drm_buddy_free_list(mm, &vres->blocks, 0);
++ gpu_buddy_free_list(mm, &vres->blocks, 0);
+ mgr->visible_avail += vres->used_visible_size;
+ mutex_unlock(&mgr->lock);
+
+@@ -227,7 +228,7 @@ static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man,
+ struct drm_printer *printer)
+ {
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
+- struct drm_buddy *mm = &mgr->mm;
++ struct gpu_buddy *mm = &mgr->mm;
+
+ mutex_lock(&mgr->lock);
+ drm_printf(printer, "default_page_size: %lluKiB\n",
+@@ -250,8 +251,8 @@ static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man,
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
+ struct xe_ttm_vram_mgr_resource *vres =
+ to_xe_ttm_vram_mgr_resource(res);
+- struct drm_buddy *mm = &mgr->mm;
+- struct drm_buddy_block *block;
++ struct gpu_buddy *mm = &mgr->mm;
++ struct gpu_buddy_block *block;
+
+ if (!place->fpfn && !place->lpfn)
+ return true;
+@@ -261,9 +262,9 @@ static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man,
+
+ list_for_each_entry(block, &vres->blocks, link) {
+ unsigned long fpfn =
+- drm_buddy_block_offset(block) >> PAGE_SHIFT;
++ gpu_buddy_block_offset(block) >> PAGE_SHIFT;
+ unsigned long lpfn = fpfn +
+- (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
++ (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
+
+ if (place->fpfn < lpfn && place->lpfn > fpfn)
+ return true;
+@@ -280,8 +281,8 @@ static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man,
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
+ struct xe_ttm_vram_mgr_resource *vres =
+ to_xe_ttm_vram_mgr_resource(res);
+- struct drm_buddy *mm = &mgr->mm;
+- struct drm_buddy_block *block;
++ struct gpu_buddy *mm = &mgr->mm;
++ struct gpu_buddy_block *block;
+
+ if (!place->fpfn && !place->lpfn)
+ return true;
+@@ -291,9 +292,9 @@ static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man,
+
+ list_for_each_entry(block, &vres->blocks, link) {
+ unsigned long fpfn =
+- drm_buddy_block_offset(block) >> PAGE_SHIFT;
++ gpu_buddy_block_offset(block) >> PAGE_SHIFT;
+ unsigned long lpfn = fpfn +
+- (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
++ (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
+
+ if (fpfn < place->fpfn || lpfn > place->lpfn)
+ return false;
+@@ -323,7 +324,7 @@ static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
+
+ WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size);
+
+- drm_buddy_fini(&mgr->mm);
++ gpu_buddy_fini(&mgr->mm);
+
+ ttm_resource_manager_cleanup(&mgr->manager);
+
+@@ -347,7 +348,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
+ mgr->visible_avail = io_size;
+
+ ttm_resource_manager_init(man, &xe->ttm, size);
+- err = drm_buddy_init(&mgr->mm, man->size, default_page_size);
++ err = gpu_buddy_init(&mgr->mm, man->size, default_page_size);
+ if (err)
+ return err;
+
+@@ -389,7 +390,7 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
+ if (!*sgt)
+ return -ENOMEM;
+
+- /* Determine the number of DRM_BUDDY blocks to export */
++ /* Determine the number of GPU_BUDDY blocks to export */
+ xe_res_first(res, offset, length, &cursor);
+ while (cursor.remaining) {
+ num_entries++;
+@@ -405,10 +406,10 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
+ sg->length = 0;
+
+ /*
+- * Walk down DRM_BUDDY blocks to populate scatterlist nodes
+- * @note: Use iterator api to get first the DRM_BUDDY block
++ * Walk down GPU_BUDDY blocks to populate scatterlist nodes
++ * @note: Use iterator api to get first the GPU_BUDDY block
+ * and the number of bytes from it. Access the following
+- * DRM_BUDDY block(s) if more buffer needs to exported
++ * GPU_BUDDY block(s) if more buffer needs to exported
+ */
+ xe_res_first(res, offset, length, &cursor);
+ for_each_sgtable_sg((*sgt), sg, i) {
+diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
+index 2d75cf1262893f..4aafe9b4a3bdfc 100644
+--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
++++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
+@@ -6,7 +6,7 @@
+ #ifndef _XE_TTM_VRAM_MGR_TYPES_H_
+ #define _XE_TTM_VRAM_MGR_TYPES_H_
+
+-#include <drm/drm_buddy.h>
++#include <linux/gpu_buddy.h>
+ #include <drm/ttm/ttm_device.h>
+
+ struct xe_mem_region;
+@@ -20,7 +20,7 @@ struct xe_ttm_vram_mgr {
+ /** @manager: Base TTM resource manager */
+ struct ttm_resource_manager manager;
+ /** @mm: DRM buddy allocator which manages the VRAM */
+- struct drm_buddy mm;
++ struct gpu_buddy mm;
+ /** @vram: ptr to details of associated VRAM region */
+ struct xe_mem_region *vram;
+ /** @visible_size: Proped size of the CPU visible portion */
+diff --git a/drivers/gpu/drm/xe/xe_wopcm.c b/drivers/gpu/drm/xe/xe_wopcm.c
+index d3a99157e52338..b7de817cce8870 100644
+--- a/drivers/gpu/drm/xe/xe_wopcm.c
++++ b/drivers/gpu/drm/xe/xe_wopcm.c
+@@ -47,9 +47,9 @@
+ */
+
+ /* Default WOPCM size is 2MB from Gen11, 1MB on previous platforms */
+-/* FIXME: Larger size require for 2 tile PVC, do a proper probe sooner or later */
++/* FIXME: Larger size require for some platforms, do a proper probe sooner or later */
+ #define DGFX_WOPCM_SIZE SZ_4M
+-/* FIXME: Larger size require for MTL, do a proper probe sooner or later */
++#define LNL_WOPCM_SIZE SZ_8M
+ #define MTL_WOPCM_SIZE SZ_4M
+ #define WOPCM_SIZE SZ_2M
+
+@@ -179,9 +179,14 @@ err_out:
+
+ u32 xe_wopcm_size(struct xe_device *xe)
+ {
+- return IS_DGFX(xe) ? DGFX_WOPCM_SIZE :
+- xe->info.platform == XE_METEORLAKE ? MTL_WOPCM_SIZE :
+- WOPCM_SIZE;
++ if (xe->info.platform >= XE_LUNARLAKE)
++ return LNL_WOPCM_SIZE;
++ else if (IS_DGFX(xe))
++ return DGFX_WOPCM_SIZE;
++ else if (xe->info.platform == XE_METEORLAKE)
++ return MTL_WOPCM_SIZE;
++ else
++ return WOPCM_SIZE;
+ }
+
+ /**
+diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
+index ef42e9a883f4f9..4dd83627fae06a 100644
+--- a/drivers/gpu/host1x/bus.c
++++ b/drivers/gpu/host1x/bus.c
+@@ -1006,10 +1006,10 @@ void host1x_bo_clear_cached_mappings(struct host1x_bo *bo)
+ if (WARN_ON(!cache))
+ continue;
+
+- mutex_lock(&mapping->cache->lock);
++ mutex_lock(&cache->lock);
+ WARN_ON(kref_read(&mapping->ref) != 1);
+ __host1x_bo_unpin(&mapping->ref);
+- mutex_unlock(&mapping->cache->lock);
++ mutex_unlock(&cache->lock);
+ }
+ }
+ EXPORT_SYMBOL(host1x_bo_clear_cached_mappings);
+diff --git a/drivers/gpu/tests/Makefile b/drivers/gpu/tests/Makefile
+new file mode 100644
+index 00000000000000..4183e6e2de4511
+--- /dev/null
++++ b/drivers/gpu/tests/Makefile
+@@ -0,0 +1,4 @@
++# SPDX-License-Identifier: GPL-2.0
++
++gpu_buddy_tests-y = gpu_buddy_test.o gpu_random.o
++obj-$(CONFIG_GPU_BUDDY_KUNIT_TEST) += gpu_buddy_tests.o
+diff --git a/drivers/gpu/tests/gpu_buddy_test.c b/drivers/gpu/tests/gpu_buddy_test.c
+new file mode 100644
+index 00000000000000..dd3367340ce8ac
+--- /dev/null
++++ b/drivers/gpu/tests/gpu_buddy_test.c
+@@ -0,0 +1,778 @@
++// SPDX-License-Identifier: MIT
++/*
++ * Copyright © 2019 Intel Corporation
++ * Copyright © 2022 MaÃra Canal <[email protected]>
++ */
++
++#include <kunit/test.h>
++
++#include <linux/prime_numbers.h>
++#include <linux/sched/signal.h>
++#include <linux/sizes.h>
++
++#include <linux/gpu_buddy.h>
++
++#include "gpu_random.h"
++
++static unsigned int random_seed;
++
++static inline u64 get_size(int order, u64 chunk_size)
++{
++ return (1 << order) * chunk_size;
++}
++
++static void drm_test_buddy_alloc_range_bias(struct kunit *test)
++{
++ u32 mm_size, size, ps, bias_size, bias_start, bias_end, bias_rem;
++ GPU_RND_STATE(prng, random_seed);
++ unsigned int i, count, *order;
++ struct gpu_buddy_block *block;
++ unsigned long flags;
++ struct gpu_buddy mm;
++ LIST_HEAD(allocated);
++
++ bias_size = SZ_1M;
++ ps = roundup_pow_of_two(prandom_u32_state(&prng) % bias_size);
++ ps = max(SZ_4K, ps);
++ mm_size = (SZ_8M-1) & ~(ps-1); /* Multiple roots */
++
++ kunit_info(test, "mm_size=%u, ps=%u\n", mm_size, ps);
++
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps),
++ "buddy_init failed\n");
++
++ count = mm_size / bias_size;
++ order = gpu_random_order(count, &prng);
++ KUNIT_EXPECT_TRUE(test, order);
++
++ /*
++ * Idea is to split the address space into uniform bias ranges, and then
++ * in some random order allocate within each bias, using various
++ * patterns within. This should detect if allocations leak out from a
++ * given bias, for example.
++ */
++
++ for (i = 0; i < count; i++) {
++ LIST_HEAD(tmp);
++ u32 size;
++
++ bias_start = order[i] * bias_size;
++ bias_end = bias_start + bias_size;
++ bias_rem = bias_size;
++
++ /* internal round_up too big */
++ KUNIT_ASSERT_TRUE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, bias_size + ps, bias_size,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, bias_size, bias_size);
++
++ /* size too big */
++ KUNIT_ASSERT_TRUE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, bias_size + ps, ps,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, bias_size + ps, ps);
++
++ /* bias range too small for size */
++ KUNIT_ASSERT_TRUE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start + ps,
++ bias_end, bias_size, ps,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start + ps, bias_end, bias_size, ps);
++
++ /* bias misaligned */
++ KUNIT_ASSERT_TRUE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start + ps,
++ bias_end - ps,
++ bias_size >> 1, bias_size >> 1,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc h didn't fail with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start + ps, bias_end - ps, bias_size >> 1, bias_size >> 1);
++
++ /* single big page */
++ KUNIT_ASSERT_FALSE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, bias_size, bias_size,
++ &tmp,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc i failed with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, bias_size, bias_size);
++ gpu_buddy_free_list(&mm, &tmp, 0);
++
++ /* single page with internal round_up */
++ KUNIT_ASSERT_FALSE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, ps, bias_size,
++ &tmp,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, ps, bias_size);
++ gpu_buddy_free_list(&mm, &tmp, 0);
++
++ /* random size within */
++ size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
++ if (size)
++ KUNIT_ASSERT_FALSE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, size, ps,
++ &tmp,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, size, ps);
++
++ bias_rem -= size;
++ /* too big for current avail */
++ KUNIT_ASSERT_TRUE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, bias_rem + ps, ps,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, bias_rem + ps, ps);
++
++ if (bias_rem) {
++ /* random fill of the remainder */
++ size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
++ size = max(size, ps);
++
++ KUNIT_ASSERT_FALSE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, size, ps,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, size, ps);
++ /*
++ * Intentionally allow some space to be left
++ * unallocated, and ideally not always on the bias
++ * boundaries.
++ */
++ gpu_buddy_free_list(&mm, &tmp, 0);
++ } else {
++ list_splice_tail(&tmp, &allocated);
++ }
++ }
++
++ kfree(order);
++ gpu_buddy_free_list(&mm, &allocated, 0);
++ gpu_buddy_fini(&mm);
++
++ /*
++ * Something more free-form. Idea is to pick a random starting bias
++ * range within the address space and then start filling it up. Also
++ * randomly grow the bias range in both directions as we go along. This
++ * should give us bias start/end which is not always uniform like above,
++ * and in some cases will require the allocator to jump over already
++ * allocated nodes in the middle of the address space.
++ */
++
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps),
++ "buddy_init failed\n");
++
++ bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
++ bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
++ bias_end = max(bias_end, bias_start + ps);
++ bias_rem = bias_end - bias_start;
++
++ do {
++ u32 size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
++
++ KUNIT_ASSERT_FALSE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, size, ps,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, size, ps);
++ bias_rem -= size;
++
++ /*
++ * Try to randomly grow the bias range in both directions, or
++ * only one, or perhaps don't grow at all.
++ */
++ do {
++ u32 old_bias_start = bias_start;
++ u32 old_bias_end = bias_end;
++
++ if (bias_start)
++ bias_start -= round_up(prandom_u32_state(&prng) % bias_start, ps);
++ if (bias_end != mm_size)
++ bias_end += round_up(prandom_u32_state(&prng) % (mm_size - bias_end), ps);
++
++ bias_rem += old_bias_start - bias_start;
++ bias_rem += bias_end - old_bias_end;
++ } while (!bias_rem && (bias_start || bias_end != mm_size));
++ } while (bias_rem);
++
++ KUNIT_ASSERT_EQ(test, bias_start, 0);
++ KUNIT_ASSERT_EQ(test, bias_end, mm_size);
++ KUNIT_ASSERT_TRUE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start, bias_end,
++ ps, ps,
++ &allocated,
++ GPU_BUDDY_RANGE_ALLOCATION),
++ "buddy_alloc passed with bias(%x-%x), size=%u\n",
++ bias_start, bias_end, ps);
++
++ gpu_buddy_free_list(&mm, &allocated, 0);
++ gpu_buddy_fini(&mm);
++
++ /*
++ * Allocate cleared blocks in the bias range when the DRM buddy's clear avail is
++ * zero. This will validate the bias range allocation in scenarios like system boot
++ * when no cleared blocks are available and exercise the fallback path too. The resulting
++ * blocks should always be dirty.
++ */
++
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps),
++ "buddy_init failed\n");
++
++ bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
++ bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
++ bias_end = max(bias_end, bias_start + ps);
++ bias_rem = bias_end - bias_start;
++
++ flags = GPU_BUDDY_CLEAR_ALLOCATION | GPU_BUDDY_RANGE_ALLOCATION;
++ size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
++
++ KUNIT_ASSERT_FALSE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, bias_start,
++ bias_end, size, ps,
++ &allocated,
++ flags),
++ "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
++ bias_start, bias_end, size, ps);
++
++ list_for_each_entry(block, &allocated, link)
++ KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false);
++
++ gpu_buddy_free_list(&mm, &allocated, 0);
++ gpu_buddy_fini(&mm);
++}
++
++static void drm_test_buddy_alloc_clear(struct kunit *test)
++{
++ unsigned long n_pages, total, i = 0;
++ GPU_RND_STATE(prng, random_seed);
++ const unsigned long ps = SZ_4K;
++ struct gpu_buddy_block *block;
++ const int max_order = 12;
++ LIST_HEAD(allocated);
++ struct gpu_buddy mm;
++ unsigned int order;
++ u32 mm_size, size;
++ LIST_HEAD(dirty);
++ LIST_HEAD(clean);
++
++ mm_size = SZ_4K << max_order;
++ KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
++
++ KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
++
++ /*
++ * Idea is to allocate and free some random portion of the address space,
++ * returning those pages as non-dirty and randomly alternate between
++ * requesting dirty and non-dirty pages (not going over the limit
++ * we freed as non-dirty), putting that into two separate lists.
++ * Loop over both lists at the end checking that the dirty list
++ * is indeed all dirty pages and vice versa. Free it all again,
++ * keeping the dirty/clear status.
++ */
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 5 * ps, ps, &allocated,
++ GPU_BUDDY_TOPDOWN_ALLOCATION),
++ "buddy_alloc hit an error size=%lu\n", 5 * ps);
++ gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED);
++
++ n_pages = 10;
++ do {
++ unsigned long flags;
++ struct list_head *list;
++ int slot = i % 2;
++
++ if (slot == 0) {
++ list = &dirty;
++ flags = 0;
++ } else {
++ list = &clean;
++ flags = GPU_BUDDY_CLEAR_ALLOCATION;
++ }
++
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ ps, ps, list,
++ flags),
++ "buddy_alloc hit an error size=%lu\n", ps);
++ } while (++i < n_pages);
++
++ list_for_each_entry(block, &clean, link)
++ KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), true);
++
++ list_for_each_entry(block, &dirty, link)
++ KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false);
++
++ gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
++
++ /*
++ * Trying to go over the clear limit for some allocation.
++ * The allocation should never fail with reasonable page-size.
++ */
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 10 * ps, ps, &clean,
++ GPU_BUDDY_CLEAR_ALLOCATION),
++ "buddy_alloc hit an error size=%lu\n", 10 * ps);
++
++ gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
++ gpu_buddy_free_list(&mm, &dirty, 0);
++ gpu_buddy_fini(&mm);
++
++ KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
++
++ /*
++ * Create a new mm. Intentionally fragment the address space by creating
++ * two alternating lists. Free both lists, one as dirty the other as clean.
++ * Try to allocate double the previous size with matching min_page_size. The
++ * allocation should never fail as it calls the force_merge. Also check that
++ * the page is always dirty after force_merge. Free the page as dirty, then
++ * repeat the whole thing, increment the order until we hit the max_order.
++ */
++
++ i = 0;
++ n_pages = mm_size / ps;
++ do {
++ struct list_head *list;
++ int slot = i % 2;
++
++ if (slot == 0)
++ list = &dirty;
++ else
++ list = &clean;
++
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ ps, ps, list, 0),
++ "buddy_alloc hit an error size=%lu\n", ps);
++ } while (++i < n_pages);
++
++ gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
++ gpu_buddy_free_list(&mm, &dirty, 0);
++
++ order = 1;
++ do {
++ size = SZ_4K << order;
++
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ size, size, &allocated,
++ GPU_BUDDY_CLEAR_ALLOCATION),
++ "buddy_alloc hit an error size=%u\n", size);
++ total = 0;
++ list_for_each_entry(block, &allocated, link) {
++ if (size != mm_size)
++ KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false);
++ total += gpu_buddy_block_size(&mm, block);
++ }
++ KUNIT_EXPECT_EQ(test, total, size);
++
++ gpu_buddy_free_list(&mm, &allocated, 0);
++ } while (++order <= max_order);
++
++ gpu_buddy_fini(&mm);
++
++ /*
++ * Create a new mm with a non power-of-two size. Allocate a random size, free as
++ * cleared and then call fini. This will ensure the multi-root force merge during
++ * fini.
++ */
++ mm_size = 12 * SZ_4K;
++ size = max(round_up(prandom_u32_state(&prng) % mm_size, ps), ps);
++ KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ size, ps, &allocated,
++ GPU_BUDDY_TOPDOWN_ALLOCATION),
++ "buddy_alloc hit an error size=%u\n", size);
++ gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED);
++ gpu_buddy_fini(&mm);
++}
++
++static void drm_test_buddy_alloc_contiguous(struct kunit *test)
++{
++ const unsigned long ps = SZ_4K, mm_size = 16 * 3 * SZ_4K;
++ unsigned long i, n_pages, total;
++ struct gpu_buddy_block *block;
++ struct gpu_buddy mm;
++ LIST_HEAD(left);
++ LIST_HEAD(middle);
++ LIST_HEAD(right);
++ LIST_HEAD(allocated);
++
++ KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
++
++ /*
++ * Idea is to fragment the address space by alternating block
++ * allocations between three different lists; one for left, middle and
++ * right. We can then free a list to simulate fragmentation. In
++ * particular we want to exercise the GPU_BUDDY_CONTIGUOUS_ALLOCATION,
++ * including the try_harder path.
++ */
++
++ i = 0;
++ n_pages = mm_size / ps;
++ do {
++ struct list_head *list;
++ int slot = i % 3;
++
++ if (slot == 0)
++ list = &left;
++ else if (slot == 1)
++ list = &middle;
++ else
++ list = &right;
++ KUNIT_ASSERT_FALSE_MSG(test,
++ gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ ps, ps, list, 0),
++ "buddy_alloc hit an error size=%lu\n",
++ ps);
++ } while (++i < n_pages);
++
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 3 * ps, ps, &allocated,
++ GPU_BUDDY_CONTIGUOUS_ALLOCATION),
++ "buddy_alloc didn't error size=%lu\n", 3 * ps);
++
++ gpu_buddy_free_list(&mm, &middle, 0);
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 3 * ps, ps, &allocated,
++ GPU_BUDDY_CONTIGUOUS_ALLOCATION),
++ "buddy_alloc didn't error size=%lu\n", 3 * ps);
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 2 * ps, ps, &allocated,
++ GPU_BUDDY_CONTIGUOUS_ALLOCATION),
++ "buddy_alloc didn't error size=%lu\n", 2 * ps);
++
++ gpu_buddy_free_list(&mm, &right, 0);
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 3 * ps, ps, &allocated,
++ GPU_BUDDY_CONTIGUOUS_ALLOCATION),
++ "buddy_alloc didn't error size=%lu\n", 3 * ps);
++ /*
++ * At this point we should have enough contiguous space for 2 blocks,
++ * however they are never buddies (since we freed middle and right) so
++ * will require the try_harder logic to find them.
++ */
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 2 * ps, ps, &allocated,
++ GPU_BUDDY_CONTIGUOUS_ALLOCATION),
++ "buddy_alloc hit an error size=%lu\n", 2 * ps);
++
++ gpu_buddy_free_list(&mm, &left, 0);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
++ 3 * ps, ps, &allocated,
++ GPU_BUDDY_CONTIGUOUS_ALLOCATION),
++ "buddy_alloc hit an error size=%lu\n", 3 * ps);
++
++ total = 0;
++ list_for_each_entry(block, &allocated, link)
++ total += gpu_buddy_block_size(&mm, block);
++
++ KUNIT_ASSERT_EQ(test, total, ps * 2 + ps * 3);
++
++ gpu_buddy_free_list(&mm, &allocated, 0);
++ gpu_buddy_fini(&mm);
++}
++
++static void drm_test_buddy_alloc_pathological(struct kunit *test)
++{
++ u64 mm_size, size, start = 0;
++ struct gpu_buddy_block *block;
++ const int max_order = 3;
++ unsigned long flags = 0;
++ int order, top;
++ struct gpu_buddy mm;
++ LIST_HEAD(blocks);
++ LIST_HEAD(holes);
++ LIST_HEAD(tmp);
++
++ /*
++ * Create a pot-sized mm, then allocate one of each possible
++ * order within. This should leave the mm with exactly one
++ * page left. Free the largest block, then whittle down again.
++ * Eventually we will have a fully 50% fragmented mm.
++ */
++
++ mm_size = SZ_4K << max_order;
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
++ "buddy_init failed\n");
++
++ KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
++
++ for (top = max_order; top; top--) {
++ /* Make room by freeing the largest allocated block */
++ block = list_first_entry_or_null(&blocks, typeof(*block), link);
++ if (block) {
++ list_del(&block->link);
++ gpu_buddy_free_block(&mm, block);
++ }
++
++ for (order = top; order--;) {
++ size = get_size(order, mm.chunk_size);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start,
++ mm_size, size, size,
++ &tmp, flags),
++ "buddy_alloc hit -ENOMEM with order=%d, top=%d\n",
++ order, top);
++
++ block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
++ KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
++
++ list_move_tail(&block->link, &blocks);
++ }
++
++ /* There should be one final page for this sub-allocation */
++ size = get_size(0, mm.chunk_size);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc hit -ENOMEM for hole\n");
++
++ block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
++ KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
++
++ list_move_tail(&block->link, &holes);
++
++ size = get_size(top, mm.chunk_size);
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!",
++ top, max_order);
++ }
++
++ gpu_buddy_free_list(&mm, &holes, 0);
++
++ /* Nothing larger than blocks of chunk_size now available */
++ for (order = 1; order <= max_order; order++) {
++ size = get_size(order, mm.chunk_size);
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc unexpectedly succeeded at order %d, it should be full!",
++ order);
++ }
++
++ list_splice_tail(&holes, &blocks);
++ gpu_buddy_free_list(&mm, &blocks, 0);
++ gpu_buddy_fini(&mm);
++}
++
++static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
++{
++ u64 mm_size, size, start = 0;
++ struct gpu_buddy_block *block, *bn;
++ const unsigned int max_order = 16;
++ unsigned long flags = 0;
++ struct gpu_buddy mm;
++ unsigned int order;
++ LIST_HEAD(blocks);
++ LIST_HEAD(tmp);
++
++ /*
++ * Create a pot-sized mm, then allocate one of each possible
++ * order within. This should leave the mm with exactly one
++ * page left.
++ */
++
++ mm_size = SZ_4K << max_order;
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
++ "buddy_init failed\n");
++
++ KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
++
++ for (order = 0; order < max_order; order++) {
++ size = get_size(order, mm.chunk_size);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc hit -ENOMEM with order=%d\n",
++ order);
++
++ block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
++ KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
++
++ list_move_tail(&block->link, &blocks);
++ }
++
++ /* And now the last remaining block available */
++ size = get_size(0, mm.chunk_size);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc hit -ENOMEM on final alloc\n");
++
++ block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
++ KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
++
++ list_move_tail(&block->link, &blocks);
++
++ /* Should be completely full! */
++ for (order = max_order; order--;) {
++ size = get_size(order, mm.chunk_size);
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc unexpectedly succeeded, it should be full!");
++ }
++
++ block = list_last_entry(&blocks, typeof(*block), link);
++ list_del(&block->link);
++ gpu_buddy_free_block(&mm, block);
++
++ /* As we free in increasing size, we make available larger blocks */
++ order = 1;
++ list_for_each_entry_safe(block, bn, &blocks, link) {
++ list_del(&block->link);
++ gpu_buddy_free_block(&mm, block);
++
++ size = get_size(order, mm.chunk_size);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc hit -ENOMEM with order=%d\n",
++ order);
++
++ block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
++ KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
++
++ list_del(&block->link);
++ gpu_buddy_free_block(&mm, block);
++ order++;
++ }
++
++ /* To confirm, now the whole mm should be available */
++ size = get_size(max_order, mm.chunk_size);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
++ max_order);
++
++ block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
++ KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
++
++ list_del(&block->link);
++ gpu_buddy_free_block(&mm, block);
++ gpu_buddy_free_list(&mm, &blocks, 0);
++ gpu_buddy_fini(&mm);
++}
++
++static void drm_test_buddy_alloc_optimistic(struct kunit *test)
++{
++ u64 mm_size, size, start = 0;
++ struct gpu_buddy_block *block;
++ unsigned long flags = 0;
++ const int max_order = 16;
++ struct gpu_buddy mm;
++ LIST_HEAD(blocks);
++ LIST_HEAD(tmp);
++ int order;
++
++ /*
++ * Create a mm with one block of each order available, and
++ * try to allocate them all.
++ */
++
++ mm_size = SZ_4K * ((1 << (max_order + 1)) - 1);
++
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
++ "buddy_init failed\n");
++
++ KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
++
++ for (order = 0; order <= max_order; order++) {
++ size = get_size(order, mm.chunk_size);
++ KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc hit -ENOMEM with order=%d\n",
++ order);
++
++ block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
++ KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
++
++ list_move_tail(&block->link, &blocks);
++ }
++
++ /* Should be completely full! */
++ size = get_size(0, mm.chunk_size);
++ KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
++ size, size, &tmp, flags),
++ "buddy_alloc unexpectedly succeeded, it should be full!");
++
++ gpu_buddy_free_list(&mm, &blocks, 0);
++ gpu_buddy_fini(&mm);
++}
++
++static void drm_test_buddy_alloc_limit(struct kunit *test)
++{
++ u64 size = U64_MAX, start = 0;
++ struct gpu_buddy_block *block;
++ unsigned long flags = 0;
++ LIST_HEAD(allocated);
++ struct gpu_buddy mm;
++
++ KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, size, SZ_4K));
++
++ KUNIT_EXPECT_EQ_MSG(test, mm.max_order, GPU_BUDDY_MAX_ORDER,
++ "mm.max_order(%d) != %d\n", mm.max_order,
++ GPU_BUDDY_MAX_ORDER);
++
++ size = mm.chunk_size << mm.max_order;
++ KUNIT_EXPECT_FALSE(test, gpu_buddy_alloc_blocks(&mm, start, size, size,
++ mm.chunk_size, &allocated, flags));
++
++ block = list_first_entry_or_null(&allocated, struct gpu_buddy_block, link);
++ KUNIT_EXPECT_TRUE(test, block);
++
++ KUNIT_EXPECT_EQ_MSG(test, gpu_buddy_block_order(block), mm.max_order,
++ "block order(%d) != %d\n",
++ gpu_buddy_block_order(block), mm.max_order);
++
++ KUNIT_EXPECT_EQ_MSG(test, gpu_buddy_block_size(&mm, block),
++ BIT_ULL(mm.max_order) * mm.chunk_size,
++ "block size(%llu) != %llu\n",
++ gpu_buddy_block_size(&mm, block),
++ BIT_ULL(mm.max_order) * mm.chunk_size);
++
++ gpu_buddy_free_list(&mm, &allocated, 0);
++ gpu_buddy_fini(&mm);
++}
++
++static int gpu_buddy_suite_init(struct kunit_suite *suite)
++{
++ while (!random_seed)
++ random_seed = get_random_u32();
++
++ kunit_info(suite, "Testing DRM buddy manager, with random_seed=0x%x\n",
++ random_seed);
++
++ return 0;
++}
++
++static struct kunit_case gpu_buddy_tests[] = {
++ KUNIT_CASE(drm_test_buddy_alloc_limit),
++ KUNIT_CASE(drm_test_buddy_alloc_optimistic),
++ KUNIT_CASE(drm_test_buddy_alloc_pessimistic),
++ KUNIT_CASE(drm_test_buddy_alloc_pathological),
++ KUNIT_CASE(drm_test_buddy_alloc_contiguous),
++ KUNIT_CASE(drm_test_buddy_alloc_clear),
++ KUNIT_CASE(drm_test_buddy_alloc_range_bias),
++ {}
++};
++
++static struct kunit_suite gpu_buddy_test_suite = {
++ .name = "gpu_buddy",
++ .suite_init = gpu_buddy_suite_init,
++ .test_cases = gpu_buddy_tests,
++};
++
++kunit_test_suite(gpu_buddy_test_suite);
++
++MODULE_AUTHOR("Intel Corporation");
++MODULE_DESCRIPTION("Kunit test for gpu_buddy functions");
++MODULE_LICENSE("GPL");
+diff --git a/drivers/gpu/tests/gpu_random.c b/drivers/gpu/tests/gpu_random.c
+new file mode 100644
+index 00000000000000..5a78a361686f46
+--- /dev/null
++++ b/drivers/gpu/tests/gpu_random.c
+@@ -0,0 +1,43 @@
++// SPDX-License-Identifier: GPL-2.0
++#include <linux/bitops.h>
++#include <linux/kernel.h>
++#include <linux/random.h>
++#include <linux/slab.h>
++#include <linux/types.h>
++
++#include "gpu_random.h"
++
++u32 gpu_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state)
++{
++ return upper_32_bits((u64)prandom_u32_state(state) * ep_ro);
++}
++EXPORT_SYMBOL(gpu_prandom_u32_max_state);
++
++void gpu_random_reorder(unsigned int *order, unsigned int count,
++ struct rnd_state *state)
++{
++ unsigned int i, j;
++
++ for (i = 0; i < count; ++i) {
++ BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32));
++ j = gpu_prandom_u32_max_state(count, state);
++ swap(order[i], order[j]);
++ }
++}
++EXPORT_SYMBOL(gpu_random_reorder);
++
++unsigned int *gpu_random_order(unsigned int count, struct rnd_state *state)
++{
++ unsigned int *order, i;
++
++ order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
++ if (!order)
++ return order;
++
++ for (i = 0; i < count; i++)
++ order[i] = i;
++
++ gpu_random_reorder(order, count, state);
++ return order;
++}
++EXPORT_SYMBOL(gpu_random_order);
+diff --git a/drivers/gpu/tests/gpu_random.h b/drivers/gpu/tests/gpu_random.h
+new file mode 100644
+index 00000000000000..c3caf120a9c937
+--- /dev/null
++++ b/drivers/gpu/tests/gpu_random.h
+@@ -0,0 +1,28 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++#ifndef __GPU_RANDOM_H__
++#define __GPU_RANDOM_H__
++
++/* This is a temporary home for a couple of utility functions that should
++ * be transposed to lib/ at the earliest convenience.
++ */
++
++#include <linux/random.h>
++
++#define GPU_RND_STATE_INITIALIZER(seed__) ({ \
++ struct rnd_state state__; \
++ prandom_seed_state(&state__, (seed__)); \
++ state__; \
++})
++
++#define GPU_RND_STATE(name__, seed__) \
++ struct rnd_state name__ = GPU_RND_STATE_INITIALIZER(seed__)
++
++unsigned int *gpu_random_order(unsigned int count,
++ struct rnd_state *state);
++void gpu_random_reorder(unsigned int *order,
++ unsigned int count,
++ struct rnd_state *state);
++u32 gpu_prandom_u32_max_state(u32 ep_ro,
++ struct rnd_state *state);
++
++#endif /* !__GPU_RANDOM_H__ */
+diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c
+index 483dfb806cc510..ad1a2ff618a38a 100644
+--- a/drivers/hwmon/asus-ec-sensors.c
++++ b/drivers/hwmon/asus-ec-sensors.c
+@@ -596,7 +596,7 @@ struct ec_sensors_data {
+ /* sorted list of unique register banks */
+ u8 banks[ASUS_EC_MAX_BANK + 1];
+ /* in jiffies */
+- unsigned long last_updated;
++ u64 next_update;
+ struct lock_data lock_data;
+ /* number of board EC sensors */
+ u8 nr_sensors;
+@@ -776,7 +776,7 @@ static int asus_ec_block_read(const struct device *dev,
+ }
+ for (ireg = 0; ireg < ec->nr_registers; ireg++) {
+ reg_bank = register_bank(ec->registers[ireg]);
+- if (reg_bank < bank) {
++ if (reg_bank != bank) {
+ continue;
+ }
+ ec_read(register_index(ec->registers[ireg]),
+@@ -865,13 +865,12 @@ static int get_cached_value_or_update(const struct device *dev,
+ int sensor_index,
+ struct ec_sensors_data *state, s32 *value)
+ {
+- if (time_after(jiffies, state->last_updated + HZ)) {
++ if (time_after64(get_jiffies_64(), state->next_update)) {
+ if (update_ec_sensors(dev, state)) {
+ dev_err(dev, "update_ec_sensors() failure\n");
+ return -EIO;
+ }
+-
+- state->last_updated = jiffies;
++ state->next_update = get_jiffies_64() + HZ;
+ }
+
+ *value = state->sensors[sensor_index].cached_value;
+@@ -989,6 +988,7 @@ static int asus_ec_probe(struct platform_device *pdev)
+ if (!ec_data)
+ return -ENOMEM;
+
++ ec_data->next_update = INITIAL_JIFFIES;
+ dev_set_drvdata(dev, ec_data);
+ ec_data->board_info = pboard_info;
+
+@@ -1064,9 +1064,11 @@ static int asus_ec_probe(struct platform_device *pdev)
+ if (!nr_count[type])
+ continue;
+
+- asus_ec_hwmon_add_chan_info(asus_ec_hwmon_chan, dev,
+- nr_count[type], type,
+- hwmon_attributes[type]);
++ status = asus_ec_hwmon_add_chan_info(asus_ec_hwmon_chan, dev,
++ nr_count[type], type,
++ hwmon_attributes[type]);
++ if (status)
++ return status;
+ *ptr_asus_ec_ci++ = asus_ec_hwmon_chan++;
+ }
+
+diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c
+index b7b911f8359c7f..71e48b3bba60f0 100644
+--- a/drivers/hwmon/corsair-cpro.c
++++ b/drivers/hwmon/corsair-cpro.c
+@@ -645,6 +645,7 @@ static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id)
+
+ out_hw_close:
+ hid_hw_close(hdev);
++ hid_device_io_stop(hdev);
+ out_hw_stop:
+ hid_hw_stop(hdev);
+ return ret;
+diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c
+index 93937e1bce196b..4ab73bcef1d125 100644
+--- a/drivers/hwmon/corsair-psu.c
++++ b/drivers/hwmon/corsair-psu.c
+@@ -831,6 +831,7 @@ static int corsairpsu_probe(struct hid_device *hdev, const struct hid_device_id
+
+ fail_and_close:
+ hid_hw_close(hdev);
++ hid_device_io_stop(hdev);
+ fail_and_stop:
+ hid_hw_stop(hdev);
+ return ret;
+diff --git a/drivers/hwmon/gigabyte_waterforce.c b/drivers/hwmon/gigabyte_waterforce.c
+index 27487e215bddff..4eea05f8b569c2 100644
+--- a/drivers/hwmon/gigabyte_waterforce.c
++++ b/drivers/hwmon/gigabyte_waterforce.c
+@@ -371,13 +371,15 @@ static int waterforce_probe(struct hid_device *hdev, const struct hid_device_id
+ if (IS_ERR(priv->hwmon_dev)) {
+ ret = PTR_ERR(priv->hwmon_dev);
+ hid_err(hdev, "hwmon registration failed with %d\n", ret);
+- goto fail_and_close;
++ goto fail_and_io_stop;
+ }
+
+ waterforce_debugfs_init(priv);
+
+ return 0;
+
++fail_and_io_stop:
++ hid_device_io_stop(hdev);
+ fail_and_close:
+ hid_hw_close(hdev);
+ fail_and_stop:
+diff --git a/drivers/hwmon/nzxt-kraken3.c b/drivers/hwmon/nzxt-kraken3.c
+index d00409bcab93ad..05525406c5fbb5 100644
+--- a/drivers/hwmon/nzxt-kraken3.c
++++ b/drivers/hwmon/nzxt-kraken3.c
+@@ -948,7 +948,7 @@ static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id
+ ret = kraken3_init_device(hdev);
+ if (ret < 0) {
+ hid_err(hdev, "device init failed with %d\n", ret);
+- goto fail_and_close;
++ goto fail_and_stop_io;
+ }
+
+ ret = kraken3_get_fw_ver(hdev);
+@@ -960,13 +960,15 @@ static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id
+ if (IS_ERR(priv->hwmon_dev)) {
+ ret = PTR_ERR(priv->hwmon_dev);
+ hid_err(hdev, "hwmon registration failed with %d\n", ret);
+- goto fail_and_close;
++ goto fail_and_stop_io;
+ }
+
+ kraken3_debugfs_init(priv, device_name);
+
+ return 0;
+
++fail_and_stop_io:
++ hid_device_io_stop(hdev);
+ fail_and_close:
+ hid_hw_close(hdev);
+ fail_and_stop:
+diff --git a/drivers/hwmon/nzxt-smart2.c b/drivers/hwmon/nzxt-smart2.c
+index c2d1173f42fefb..6f8febda4277c9 100644
+--- a/drivers/hwmon/nzxt-smart2.c
++++ b/drivers/hwmon/nzxt-smart2.c
+@@ -774,7 +774,7 @@ static int nzxt_smart2_hid_probe(struct hid_device *hdev,
+
+ out_hw_close:
+ hid_hw_close(hdev);
+-
++ hid_device_io_stop(hdev);
+ out_hw_stop:
+ hid_hw_stop(hdev);
+ return ret;
+diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c
+index e18e80e832fd3f..175208d712b06e 100644
+--- a/drivers/hwmon/occ/common.c
++++ b/drivers/hwmon/occ/common.c
+@@ -1052,32 +1052,49 @@ static int occ_setup_sensor_attrs(struct occ *occ)
+ }
+
+ /* only need to do this once at startup, as OCC won't change sensors on us */
+-static void occ_parse_poll_response(struct occ *occ)
++static int occ_parse_poll_response(struct occ *occ)
+ {
+ unsigned int i, old_offset, offset = 0, size = 0;
++ u16 data_length;
+ struct occ_sensor *sensor;
+- struct occ_sensors *sensors = &occ->sensors;
++ struct occ_sensors parsed = {};
++ struct occ_sensors *sensors = &parsed;
+ struct occ_response *resp = &occ->resp;
+ struct occ_poll_response *poll =
+ (struct occ_poll_response *)&resp->data[0];
+ struct occ_poll_response_header *header = &poll->header;
+ struct occ_sensor_data_block *block = &poll->block;
+
++ data_length = get_unaligned_be16(&resp->data_length);
++ if (data_length < sizeof(*header) || data_length > OCC_RESP_DATA_BYTES) {
++ dev_err(occ->bus_dev, "invalid OCC poll response length %u\n",
++ data_length);
++ return -EMSGSIZE;
++ }
++
+ dev_info(occ->bus_dev, "OCC found, code level: %.16s\n",
+ header->occ_code_level);
+
+ for (i = 0; i < header->num_sensor_data_blocks; ++i) {
+ block = (struct occ_sensor_data_block *)((u8 *)block + offset);
++ if (size + sizeof(*header) + sizeof(block->header) >
++ data_length) {
++ dev_err(occ->bus_dev,
++ "truncated OCC sensor block header\n");
++ return -EMSGSIZE;
++ }
++
+ old_offset = offset;
+ offset = (block->header.num_sensors *
+ block->header.sensor_length) + sizeof(block->header);
+- size += offset;
+
+ /* validate all the length/size fields */
+- if ((size + sizeof(*header)) >= OCC_RESP_DATA_BYTES) {
+- dev_warn(occ->bus_dev, "exceeded response buffer\n");
+- return;
++ if (size + sizeof(*header) + offset > data_length) {
++ dev_err(occ->bus_dev,
++ "exceeded OCC poll response length\n");
++ return -EMSGSIZE;
+ }
++ size += offset;
+
+ dev_dbg(occ->bus_dev, " %04x..%04x: %.4s (%d sensors)\n",
+ old_offset, offset - 1, block->header.eye_catcher,
+@@ -1107,6 +1124,9 @@ static void occ_parse_poll_response(struct occ *occ)
+
+ dev_dbg(occ->bus_dev, "Max resp size: %u+%zd=%zd\n", size,
+ sizeof(*header), size + sizeof(*header));
++ occ->sensors = parsed;
++
++ return 0;
+ }
+
+ int occ_active(struct occ *occ, bool active)
+@@ -1138,10 +1158,12 @@ int occ_active(struct occ *occ, bool active)
+ goto unlock;
+ }
+
+- occ->active = true;
+ occ->next_update = jiffies + OCC_UPDATE_FREQUENCY;
+- occ_parse_poll_response(occ);
++ rc = occ_parse_poll_response(occ);
++ if (rc)
++ goto unlock;
+
++ occ->active = true;
+ rc = occ_setup_sensor_attrs(occ);
+ if (rc) {
+ dev_err(occ->bus_dev,
+diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c
+index 06c30fdb1954bb..4af6c53e8c8aa6 100644
+--- a/drivers/hwtracing/intel_th/core.c
++++ b/drivers/hwtracing/intel_th/core.c
+@@ -843,18 +843,8 @@ out_put_device:
+ return err;
+ }
+
+-static int intel_th_output_release(struct inode *inode, struct file *file)
+-{
+- struct intel_th_device *thdev = file->private_data;
+-
+- put_device(&thdev->dev);
+-
+- return 0;
+-}
+-
+ static const struct file_operations intel_th_output_fops = {
+ .open = intel_th_output_open,
+- .release = intel_th_output_release,
+ .llseek = noop_llseek,
+ };
+
+diff --git a/drivers/hwtracing/intel_th/msu.c b/drivers/hwtracing/intel_th/msu.c
+index 93b65a9731d721..c1418ee219bd9c 100644
+--- a/drivers/hwtracing/intel_th/msu.c
++++ b/drivers/hwtracing/intel_th/msu.c
+@@ -1482,8 +1482,10 @@ static int intel_th_msc_release(struct inode *inode, struct file *file)
+ {
+ struct msc_iter *iter = file->private_data;
+ struct msc *msc = iter->msc;
++ struct intel_th_device *thdev = msc->thdev;
+
+ msc_iter_remove(iter, msc);
++ put_device(&thdev->dev);
+
+ return 0;
+ }
+diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
+index c4fb5e9ab50611..fa51f6843932b9 100644
+--- a/drivers/i2c/busses/i2c-davinci.c
++++ b/drivers/i2c/busses/i2c-davinci.c
+@@ -864,13 +864,15 @@ static int davinci_i2c_probe(struct platform_device *pdev)
+ adap->nr = pdev->id;
+ r = i2c_add_numbered_adapter(adap);
+ if (r)
+- goto err_unuse_clocks;
++ goto err_cpufreq;
+
+ pm_runtime_mark_last_busy(dev->dev);
+ pm_runtime_put_autosuspend(dev->dev);
+
+ return 0;
+
++err_cpufreq:
++ i2c_davinci_cpufreq_deregister(dev);
+ err_unuse_clocks:
+ pm_runtime_dont_use_autosuspend(dev->dev);
+ pm_runtime_put_sync(dev->dev);
+diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
+index 24363acfc3f8c2..321fed8117222a 100644
+--- a/drivers/i2c/busses/i2c-i801.c
++++ b/drivers/i2c/busses/i2c-i801.c
+@@ -930,13 +930,13 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr,
+ */
+ if (hwpec)
+ outb_p(inb_p(SMBAUXCTL(priv)) & ~SMBAUXCTL_CRC, SMBAUXCTL(priv));
+-out:
+ /*
+ * Unlock the SMBus device for use by BIOS/ACPI,
+ * and clear status flags if not done already.
+ */
+ outb_p(SMBHSTSTS_INUSE_STS | STATUS_FLAGS, SMBHSTSTS(priv));
+
++out:
+ pm_runtime_mark_last_busy(&priv->pci_dev->dev);
+ pm_runtime_put_autosuspend(&priv->pci_dev->dev);
+ mutex_unlock(&priv->acpi_lock);
+diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
+index c5224d43eea45e..45e71120fdc146 100644
+--- a/drivers/i2c/busses/i2c-imx.c
++++ b/drivers/i2c/busses/i2c-imx.c
+@@ -1007,6 +1007,43 @@ static int i2c_imx_dma_write(struct imx_i2c_struct *i2c_imx,
+ return i2c_imx_acked(i2c_imx);
+ }
+
++static int i2c_imx_prepare_read(struct imx_i2c_struct *i2c_imx,
++ struct i2c_msg *msgs, bool atomic,
++ bool use_dma)
++{
++ int result;
++ unsigned int temp = 0;
++
++ /* write slave address */
++ imx_i2c_write_reg(i2c_8bit_addr_from_msg(msgs), i2c_imx, IMX_I2C_I2DR);
++ result = i2c_imx_trx_complete(i2c_imx, atomic);
++ if (result)
++ return result;
++ result = i2c_imx_acked(i2c_imx);
++ if (result)
++ return result;
++
++ dev_dbg(&i2c_imx->adapter.dev, "<%s> setup bus\n", __func__);
++
++ /* setup bus to read data */
++ temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR);
++ temp &= ~I2CR_MTX;
++
++ /*
++ * Reset the I2CR_TXAK flag initially for SMBus block read since the
++ * length is unknown
++ */
++ if (msgs->len - 1)
++ temp &= ~I2CR_TXAK;
++ if (use_dma)
++ temp |= I2CR_DMAEN;
++
++ imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR);
++ imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); /* dummy read */
++
++ return 0;
++}
++
+ static int i2c_imx_dma_read(struct imx_i2c_struct *i2c_imx,
+ struct i2c_msg *msgs, bool is_lastmsg)
+ {
+@@ -1017,6 +1054,11 @@ static int i2c_imx_dma_read(struct imx_i2c_struct *i2c_imx,
+ struct imx_i2c_dma *dma = i2c_imx->dma;
+ struct device *dev = &i2c_imx->adapter.dev;
+
++ result = i2c_imx_prepare_read(i2c_imx, msgs, false, true);
++ if (result)
++ return result;
++
++ dev_dbg(&i2c_imx->adapter.dev, "<%s> read data\n", __func__);
+
+ dma->chan_using = dma->chan_rx;
+ dma->dma_transfer_dir = DMA_DEV_TO_MEM;
+@@ -1127,50 +1169,25 @@ static int i2c_imx_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs,
+ return 0;
+ }
+
++static int i2c_imx_atomic_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs)
++{
++ return i2c_imx_write(i2c_imx, msgs, true);
++}
++
+ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs,
+ bool is_lastmsg, bool atomic)
+ {
+ int i, result;
+ unsigned int temp;
+ int block_data = msgs->flags & I2C_M_RECV_LEN;
+- int use_dma = i2c_imx->dma && msgs->flags & I2C_M_DMA_SAFE &&
+- msgs->len >= DMA_THRESHOLD && !block_data;
+-
+- dev_dbg(&i2c_imx->adapter.dev,
+- "<%s> write slave address: addr=0x%x\n",
+- __func__, i2c_8bit_addr_from_msg(msgs));
++ int block_err = 0;
+
+- /* write slave address */
+- imx_i2c_write_reg(i2c_8bit_addr_from_msg(msgs), i2c_imx, IMX_I2C_I2DR);
+- result = i2c_imx_trx_complete(i2c_imx, atomic);
++ result = i2c_imx_prepare_read(i2c_imx, msgs, atomic, false);
+ if (result)
+ return result;
+- result = i2c_imx_acked(i2c_imx);
+- if (result)
+- return result;
+-
+- dev_dbg(&i2c_imx->adapter.dev, "<%s> setup bus\n", __func__);
+-
+- /* setup bus to read data */
+- temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR);
+- temp &= ~I2CR_MTX;
+-
+- /*
+- * Reset the I2CR_TXAK flag initially for SMBus block read since the
+- * length is unknown
+- */
+- if ((msgs->len - 1) || block_data)
+- temp &= ~I2CR_TXAK;
+- if (use_dma)
+- temp |= I2CR_DMAEN;
+- imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR);
+- imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); /* dummy read */
+
+ dev_dbg(&i2c_imx->adapter.dev, "<%s> read data\n", __func__);
+
+- if (use_dma)
+- return i2c_imx_dma_read(i2c_imx, msgs, is_lastmsg);
+-
+ /* read data */
+ for (i = 0; i < msgs->len; i++) {
+ u8 len = 0;
+@@ -1185,8 +1202,20 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs,
+ */
+ if ((!i) && block_data) {
+ len = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR);
+- if ((len == 0) || (len > I2C_SMBUS_BLOCK_MAX))
+- return -EPROTO;
++ if ((len == 0) || (len > I2C_SMBUS_BLOCK_MAX)) {
++ /*
++ * SMBus 3.1 6.5.7: support count byte of 0.
++ * I2C_SMBUS_BLOCK_MAX case should not hold the SDA either.
++ */
++ if (len > I2C_SMBUS_BLOCK_MAX)
++ block_err = -EPROTO;
++ temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR);
++ temp |= I2CR_TXAK;
++ imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR);
++ msgs->buf[0] = 0;
++ msgs->len = 2;
++ continue;
++ }
+ dev_dbg(&i2c_imx->adapter.dev,
+ "<%s> read length: 0x%X\n",
+ __func__, len);
+@@ -1234,7 +1263,13 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs,
+ "<%s> read byte: B%d=0x%X\n",
+ __func__, i, msgs->buf[i]);
+ }
+- return 0;
++ return block_err;
++}
++
++static int i2c_imx_atomic_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs,
++ bool is_lastmsg)
++{
++ return i2c_imx_read(i2c_imx, msgs, is_lastmsg, true);
+ }
+
+ static int i2c_imx_xfer_common(struct i2c_adapter *adapter,
+@@ -1244,6 +1279,7 @@ static int i2c_imx_xfer_common(struct i2c_adapter *adapter,
+ int result;
+ bool is_lastmsg = false;
+ struct imx_i2c_struct *i2c_imx = i2c_get_adapdata(adapter);
++ int use_dma = 0;
+
+ /* Start I2C transfer */
+ result = i2c_imx_start(i2c_imx, atomic);
+@@ -1296,15 +1332,25 @@ static int i2c_imx_xfer_common(struct i2c_adapter *adapter,
+ (temp & I2SR_SRW ? 1 : 0), (temp & I2SR_IIF ? 1 : 0),
+ (temp & I2SR_RXAK ? 1 : 0));
+ #endif
++
++ use_dma = i2c_imx->dma && msgs[i].len >= DMA_THRESHOLD &&
++ msgs[i].flags & I2C_M_DMA_SAFE;
+ if (msgs[i].flags & I2C_M_RD) {
+- result = i2c_imx_read(i2c_imx, &msgs[i], is_lastmsg, atomic);
++ int block_data = msgs->flags & I2C_M_RECV_LEN;
++
++ if (atomic)
++ result = i2c_imx_atomic_read(i2c_imx, &msgs[i], is_lastmsg);
++ else if (use_dma && !block_data)
++ result = i2c_imx_dma_read(i2c_imx, &msgs[i], is_lastmsg);
++ else
++ result = i2c_imx_read(i2c_imx, &msgs[i], is_lastmsg, false);
+ } else {
+- if (!atomic &&
+- i2c_imx->dma && msgs[i].len >= DMA_THRESHOLD &&
+- msgs[i].flags & I2C_M_DMA_SAFE)
++ if (atomic)
++ result = i2c_imx_atomic_write(i2c_imx, &msgs[i]);
++ else if (use_dma)
+ result = i2c_imx_dma_write(i2c_imx, &msgs[i]);
+ else
+- result = i2c_imx_write(i2c_imx, &msgs[i], atomic);
++ result = i2c_imx_write(i2c_imx, &msgs[i], false);
+ }
+ if (result)
+ goto fail0;
+diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
+index 52d70eeb0c52ec..a225eae78d3e58 100644
+--- a/drivers/infiniband/core/cma.c
++++ b/drivers/infiniband/core/cma.c
+@@ -5231,7 +5231,7 @@ static int cma_netevent_callback(struct notifier_block *self,
+
+ list_for_each_entry(current_id, &ips_node->id_list, id_list_entry) {
+ if (!memcmp(current_id->id.route.addr.dev_addr.dst_dev_addr,
+- neigh->ha, ETH_ALEN))
++ neigh->ha, neigh->dev->addr_len))
+ continue;
+ cma_id_get(current_id);
+ if (!queue_work(cma_wq, ¤t_id->id.net_work))
+diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
+index 73f3a0b9a54b5f..96c1fd8039fb59 100644
+--- a/drivers/infiniband/core/mad.c
++++ b/drivers/infiniband/core/mad.c
+@@ -1778,6 +1778,24 @@ void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
+ &mad_send_wr->mad_agent_priv->done_list);
+ }
+
++static bool is_kernel_rmpp_data_response(struct ib_mad_agent_private *agent,
++ struct ib_mad_recv_wc *mad_recv_wc)
++{
++ const struct ib_mad_hdr *mad_hdr = &mad_recv_wc->recv_buf.mad->mad_hdr;
++ struct ib_rmpp_mad *rmpp_mad;
++
++ if (!ib_mad_kernel_rmpp_agent(&agent->agent) ||
++ !ib_response_mad(mad_hdr) ||
++ !ib_is_mad_class_rmpp(mad_hdr->mgmt_class))
++ return false;
++
++ rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad;
++
++ return (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
++ IB_MGMT_RMPP_FLAG_ACTIVE) &&
++ rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA;
++}
++
+ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
+ struct ib_mad_recv_wc *mad_recv_wc)
+ {
+@@ -1796,6 +1814,18 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
+ }
+
+ list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
++ if (is_kernel_rmpp_data_response(mad_agent_priv, mad_recv_wc)) {
++ spin_lock_irqsave(&mad_agent_priv->lock, flags);
++ mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);
++ spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
++
++ if (!mad_send_wr) {
++ ib_free_recv_mad(mad_recv_wc);
++ deref_mad_agent(mad_agent_priv);
++ return;
++ }
++ }
++
+ if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
+ mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv,
+ mad_recv_wc);
+diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c
+index f16491bd5f2b11..1206993e2e3e83 100644
+--- a/drivers/infiniband/core/umem_dmabuf.c
++++ b/drivers/infiniband/core/umem_dmabuf.c
+@@ -206,6 +206,10 @@ static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach)
+
+ if (umem_dmabuf->revoked)
+ return;
++
++ if (umem_dmabuf->pinned_revoke)
++ umem_dmabuf->pinned_revoke(umem_dmabuf->private);
++
+ ib_umem_dmabuf_unmap_pages(umem_dmabuf);
+ if (umem_dmabuf->pinned) {
+ dma_buf_unpin(umem_dmabuf->attach);
+@@ -214,6 +218,11 @@ static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach)
+ umem_dmabuf->revoked = 1;
+ }
+
++static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = {
++ .allow_peer2peer = true,
++ .move_notify = ib_umem_dmabuf_revoke_locked,
++};
++
+ static struct ib_umem_dmabuf *
+ ib_umem_dmabuf_get_pinned_and_lock(struct ib_device *device,
+ struct device *dma_device,
+@@ -266,6 +275,58 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
+ }
+ EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_with_dma_device);
+
++/**
++ * ib_umem_dmabuf_get_pinned_revocable_and_lock - Map & pin a revocable dmabuf
++ * @device: IB device.
++ * @offset: Start offset.
++ * @size: Length.
++ * @fd: dmabuf fd.
++ * @access: Access flags.
++ *
++ * Obtains a umem from a dmabuf for drivers/devices that can support revocation.
++ *
++ * Returns with dma_resv_lock held upon success. The driver must set the revoke
++ * callback prior to unlock by calling ib_umem_dmabuf_set_revoke_locked().
++ *
++ * When a revocation occurs, the revoke callback will be called. The driver must
++ * ensure that the region is no longer accessed when the callback returns. Any
++ * subsequent access attempts should also probably cause an AE for MRs.
++ *
++ * If the umem is used for an MR, the driver must ensure that the key remains in
++ * use such that it cannot be obtained by a new region until this region is
++ * fully deregistered (i.e., ibv_dereg_mr). If a driver needs to serialize with
++ * revoke calls, it can use dma_resv_lock.
++ *
++ * If successful, then the revoke callback may be called at any time and will
++ * also be called automatically upon ib_umem_release (serialized). The revoke
++ * callback will be called one time at most.
++ *
++ * Return: A pointer to ib_umem_dmabuf on success, or an ERR_PTR on failure.
++ */
++struct ib_umem_dmabuf *
++ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device,
++ unsigned long offset, size_t size,
++ int fd, int access)
++{
++ const struct dma_buf_attach_ops *ops =
++ &ib_umem_dmabuf_attach_pinned_revocable_ops;
++
++ return ib_umem_dmabuf_get_pinned_and_lock(device, device->dma_device,
++ offset, size, fd, access,
++ ops);
++}
++EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_revocable_and_lock);
++
++void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf,
++ void (*revoke)(void *priv), void *priv)
++{
++ dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
++
++ umem_dmabuf->pinned_revoke = revoke;
++ umem_dmabuf->private = priv;
++}
++EXPORT_SYMBOL(ib_umem_dmabuf_set_revoke_locked);
++
+ struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
+ unsigned long offset,
+ size_t size, int fd,
+diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c
+index 4d1f9114cd97c5..4dfab0c49e5192 100644
+--- a/drivers/infiniband/hw/erdma/erdma_qp.c
++++ b/drivers/infiniband/hw/erdma/erdma_qp.c
+@@ -573,7 +573,7 @@ int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
+ const struct ib_recv_wr *wr = recv_wr;
+ struct erdma_qp *qp = to_eqp(ibqp);
+ unsigned long flags;
+- int ret;
++ int ret = 0;
+
+ spin_lock_irqsave(&qp->lock, flags);
+
+diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
+index 1680d0ac071ed1..ae7a3939124f23 100644
+--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
++++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
+@@ -842,7 +842,7 @@ static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
+ mhop.bt_chunk_size;
+
+ for (i = 0; i < table->num_hem; ++i) {
+- obj = i * buf_chunk_size / table->obj_size;
++ obj = (u64)i * buf_chunk_size / table->obj_size;
+ if (table->hem[i])
+ hns_roce_table_mhop_put(hr_dev, table, obj, 0);
+ }
+diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
+index 92b56338819ef8..d8e101fc74ff73 100644
+--- a/drivers/infiniband/hw/irdma/verbs.c
++++ b/drivers/infiniband/hw/irdma/verbs.c
+@@ -2367,7 +2367,7 @@ static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
+ u32 pg_idx;
+
+ for (pg_idx = 0; pg_idx < npages; pg_idx++) {
+- if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])
++ if ((*arr + ((u64)pg_size * pg_idx)) != arr[pg_idx])
+ return false;
+ }
+
+@@ -2400,7 +2400,7 @@ static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc,
+
+ for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
+ arr = leaf->addr;
+- if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)
++ if ((*start_addr + ((u64)i * pg_size * PBLE_PER_PAGE)) != *arr)
+ return false;
+ ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size);
+ if (!ret)
+@@ -3252,6 +3252,9 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags,
+ if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
+ return ERR_PTR(-EOPNOTSUPP);
+
++ if (iwmr->type != IRDMA_MEMREG_TYPE_MEM)
++ return ERR_PTR(-EINVAL);
++
+ ret = ib_umem_check_rereg(iwmr->region, flags, new_access);
+ if (ret)
+ return ERR_PTR(ret);
+diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
+index d0c0cde09f1186..8ecff632ab6f3d 100644
+--- a/drivers/infiniband/sw/siw/siw_verbs.c
++++ b/drivers/infiniband/sw/siw/siw_verbs.c
+@@ -317,6 +317,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+ struct siw_ucontext *uctx =
+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
+ base_ucontext);
++ struct siw_uresp_create_qp uresp = {};
+ unsigned long flags;
+ int num_sqe, num_rqe, rv = 0;
+ size_t length;
+@@ -370,11 +371,6 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+ spin_lock_init(&qp->rq_lock);
+ spin_lock_init(&qp->orq_lock);
+
+- rv = siw_qp_add(sdev, qp);
+- if (rv)
+- goto err_atomic;
+-
+-
+ /* All queue indices are derived from modulo operations
+ * on a free running 'get' (consumer) and 'put' (producer)
+ * unsigned counter. Having queue sizes at power of two
+@@ -392,14 +388,14 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+
+ if (qp->sendq == NULL) {
+ rv = -ENOMEM;
+- goto err_out_xa;
++ goto err_out;
+ }
+ if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
+ if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
+ qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
+ else {
+ rv = -EINVAL;
+- goto err_out_xa;
++ goto err_out;
+ }
+ }
+ qp->pd = pd;
+@@ -425,7 +421,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+
+ if (qp->recvq == NULL) {
+ rv = -ENOMEM;
+- goto err_out_xa;
++ goto err_out;
+ }
+ qp->attrs.rq_size = num_rqe;
+ }
+@@ -440,11 +436,8 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+ qp->attrs.state = SIW_QP_STATE_IDLE;
+
+ if (udata) {
+- struct siw_uresp_create_qp uresp = {};
+-
+ uresp.num_sqe = num_sqe;
+ uresp.num_rqe = num_rqe;
+- uresp.qp_id = qp_id(qp);
+
+ if (qp->sendq) {
+ length = num_sqe * sizeof(struct siw_sqe);
+@@ -453,7 +446,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+ length, &uresp.sq_key);
+ if (!qp->sq_entry) {
+ rv = -ENOMEM;
+- goto err_out_xa;
++ goto err_out;
+ }
+ }
+
+@@ -465,9 +458,23 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+ if (!qp->rq_entry) {
+ uresp.sq_key = SIW_INVAL_UOBJ_KEY;
+ rv = -ENOMEM;
+- goto err_out_xa;
++ goto err_out;
+ }
+ }
++ }
++ qp->tx_cpu = siw_get_tx_cpu(sdev);
++ if (qp->tx_cpu < 0) {
++ rv = -EINVAL;
++ goto err_out;
++ }
++ init_completion(&qp->qp_free);
++
++ rv = siw_qp_add(sdev, qp);
++ if (rv)
++ goto err_out_tx;
++
++ if (udata) {
++ uresp.qp_id = qp_id(qp);
+
+ if (udata->outlen < sizeof(uresp)) {
+ rv = -EINVAL;
+@@ -477,22 +484,19 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
+ if (rv)
+ goto err_out_xa;
+ }
+- qp->tx_cpu = siw_get_tx_cpu(sdev);
+- if (qp->tx_cpu < 0) {
+- rv = -EINVAL;
+- goto err_out_xa;
+- }
++
+ INIT_LIST_HEAD(&qp->devq);
+ spin_lock_irqsave(&sdev->lock, flags);
+ list_add_tail(&qp->devq, &sdev->qp_list);
+ spin_unlock_irqrestore(&sdev->lock, flags);
+
+- init_completion(&qp->qp_free);
+-
+ return 0;
+
+ err_out_xa:
+ xa_erase(&sdev->qp_xa, qp_id(qp));
++err_out_tx:
++ siw_put_tx_cpu(qp->tx_cpu);
++err_out:
+ if (uctx) {
+ rdma_user_mmap_entry_remove(qp->sq_entry);
+ rdma_user_mmap_entry_remove(qp->rq_entry);
+diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c
+index 4073e66e4d629e..4cfcf6e945bba5 100644
+--- a/drivers/input/misc/ims-pcu.c
++++ b/drivers/input/misc/ims-pcu.c
+@@ -448,6 +448,14 @@ static void ims_pcu_handle_response(struct ims_pcu *pcu)
+ }
+ }
+
++static void ims_pcu_reset_packet(struct ims_pcu *pcu)
++{
++ pcu->have_stx = false;
++ pcu->have_dle = false;
++ pcu->read_pos = 0;
++ pcu->check_sum = 0;
++}
++
+ static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb)
+ {
+ int i;
+@@ -460,6 +468,14 @@ static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb)
+ continue;
+
+ if (pcu->have_dle) {
++ if (pcu->read_pos >= IMS_PCU_BUF_SIZE) {
++ dev_warn(pcu->dev,
++ "Packet too long (%d bytes), discarding\n",
++ pcu->read_pos);
++ ims_pcu_reset_packet(pcu);
++ continue;
++ }
++
+ pcu->have_dle = false;
+ pcu->read_buf[pcu->read_pos++] = data;
+ pcu->check_sum += data;
+@@ -472,10 +488,8 @@ static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb)
+ dev_warn(pcu->dev,
+ "Unexpected STX at byte %d, discarding old data\n",
+ pcu->read_pos);
++ ims_pcu_reset_packet(pcu);
+ pcu->have_stx = true;
+- pcu->have_dle = false;
+- pcu->read_pos = 0;
+- pcu->check_sum = 0;
+ break;
+
+ case IMS_PCU_PROTOCOL_DLE:
+@@ -495,12 +509,18 @@ static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb)
+ ims_pcu_handle_response(pcu);
+ }
+
+- pcu->have_stx = false;
+- pcu->have_dle = false;
+- pcu->read_pos = 0;
++ ims_pcu_reset_packet(pcu);
+ break;
+
+ default:
++ if (pcu->read_pos >= IMS_PCU_BUF_SIZE) {
++ dev_warn(pcu->dev,
++ "Packet too long (%d bytes), discarding\n",
++ pcu->read_pos);
++ ims_pcu_reset_packet(pcu);
++ continue;
++ }
++
+ pcu->read_buf[pcu->read_pos++] = data;
+ pcu->check_sum += data;
+ break;
+diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
+index 7fa340281b9fb2..36a814b6c4de56 100644
+--- a/drivers/iommu/amd/init.c
++++ b/drivers/iommu/amd/init.c
+@@ -3696,6 +3696,12 @@ not_found:
+ return 1;
+
+ found:
++ if (early_acpihid_map_size == EARLY_MAP_SIZE) {
++ pr_err("Early ACPI HID map overflow - ignoring ivrs_acpihid%s\n",
++ str);
++ return 1;
++ }
++
+ p = acpiid;
+ hid = strsep(&p, ":");
+ uid = p;
+diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
+index b4052fa9cc76bc..3f109a99a9e401 100644
+--- a/drivers/iommu/amd/iommu.c
++++ b/drivers/iommu/amd/iommu.c
+@@ -1424,11 +1424,23 @@ static int iommu_completion_wait(struct amd_iommu *iommu)
+ int ret;
+ u64 data;
+
+- if (!iommu->need_sync)
+- return 0;
+-
+ raw_spin_lock_irqsave(&iommu->lock, flags);
+
++ if (!iommu->need_sync) {
++ /*
++ * No command has been queued since the last completion-wait.
++ * A concurrent CPU may have already queued that CWAIT and
++ * cleared need_sync; need_sync == false only means a covering
++ * CWAIT is queued, not that all prior commands have completed.
++ * Wait for the last allocated sequence number so that any
++ * command queued before this call (possibly on another CPU)
++ * is guaranteed to have completed before returning.
++ */
++ data = iommu->cmd_sem_val;
++ raw_spin_unlock_irqrestore(&iommu->lock, flags);
++ return wait_on_sem(iommu, data);
++ }
++
+ data = get_cmdsem_val(iommu);
+ build_completion_wait(&cmd, iommu, data);
+
+@@ -1438,9 +1450,7 @@ static int iommu_completion_wait(struct amd_iommu *iommu)
+ if (ret)
+ return ret;
+
+- ret = wait_on_sem(iommu, data);
+-
+- return ret;
++ return wait_on_sem(iommu, data);
+ }
+
+ static void domain_flush_complete(struct protection_domain *domain)
+diff --git a/drivers/iommu/intel/perf.c b/drivers/iommu/intel/perf.c
+index dceeadc3ee7cdd..6afb55073e9f10 100644
+--- a/drivers/iommu/intel/perf.c
++++ b/drivers/iommu/intel/perf.c
+@@ -63,7 +63,7 @@ void dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type)
+ return;
+
+ spin_lock_irqsave(&latency_lock, flags);
+- memset(&lstat[type], 0, sizeof(*lstat) * DMAR_LATENCY_NUM);
++ memset(&lstat[type], 0, sizeof(*lstat));
+ spin_unlock_irqrestore(&latency_lock, flags);
+ }
+
+diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
+index 4a2bd65614ad2d..0278aa7df870c6 100644
+--- a/drivers/iommu/intel/svm.c
++++ b/drivers/iommu/intel/svm.c
+@@ -27,7 +27,7 @@
+
+ void intel_svm_check(struct intel_iommu *iommu)
+ {
+- if (!pasid_supported(iommu))
++ if (!pasid_supported(iommu) || !ecap_smpwc(iommu->ecap))
+ return;
+
+ if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
+diff --git a/drivers/iommu/iommufd/fault.c b/drivers/iommu/iommufd/fault.c
+index 8226e28d79a510..74e3c93bce6eae 100644
+--- a/drivers/iommu/iommufd/fault.c
++++ b/drivers/iommu/iommufd/fault.c
+@@ -270,9 +270,14 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
+
+ mutex_lock(&fault->mutex);
+ while ((group = iommufd_fault_deliver_fetch(fault))) {
++ size_t group_done = done;
++
+ if (done >= count ||
+ group->fault_count * fault_size > count - done) {
+ iommufd_fault_deliver_restore(fault, group);
++ /* Read count doesn't fit the first fault group */
++ if (done == 0)
++ rc = -EINVAL;
+ break;
+ }
+
+@@ -288,14 +293,17 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
+ iommufd_compose_fault_message(&iopf->fault,
+ &data, idev,
+ group->cookie);
+- if (copy_to_user(buf + done, &data, fault_size)) {
++ if (copy_to_user(buf + group_done, &data, fault_size)) {
+ xa_erase(&fault->response, group->cookie);
+ iommufd_fault_deliver_restore(fault, group);
+ rc = -EFAULT;
+ break;
+ }
+- done += fault_size;
++ group_done += fault_size;
+ }
++ if (rc)
++ break;
++ done = group_done;
+ }
+ mutex_unlock(&fault->mutex);
+
+diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
+index b661be629da260..0a7be46701a9c2 100644
+--- a/drivers/md/dm-integrity.c
++++ b/drivers/md/dm-integrity.c
+@@ -2502,7 +2502,7 @@ skip_spinlock:
+ struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
+ const char *mem = bvec_kmap_local(&bv);
+ if (ic->tag_size < ic->tuple_size)
+- memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size);
++ memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tag_size);
+ integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, dio->integrity_payload + pos);
+ kunmap_local(mem);
+ pos += ic->tuple_size;
+diff --git a/drivers/md/dm.c b/drivers/md/dm.c
+index ec48fcdb19ed83..269959c9204f60 100644
+--- a/drivers/md/dm.c
++++ b/drivers/md/dm.c
+@@ -737,7 +737,16 @@ static struct table_device *open_table_device(struct mapped_device *md,
+ return ERR_PTR(-ENOMEM);
+ refcount_set(&td->count, 1);
+
+- bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL);
++ /*
++ * Open the backing device with kernel rather than caller
++ * credentials. Otherwise the caller's credentials would be
++ * pinned in bdev_file->f_cred until the table device is closed.
++ * That would keep the caller's thread keyring alive long beyond the
++ * lifetime of the caller, breaking userspace expectation (e.g.
++ * cryptsetup(8) leaking the LUKS volume key).
++ */
++ scoped_with_kernel_creds()
++ bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL);
+ if (IS_ERR(bdev_file)) {
+ r = PTR_ERR(bdev_file);
+ goto out_free_td;
+diff --git a/drivers/media/cec/platform/seco/seco-cec.c b/drivers/media/cec/platform/seco/seco-cec.c
+index 5d4c5a2cae097e..8f5991cfae5e06 100644
+--- a/drivers/media/cec/platform/seco/seco-cec.c
++++ b/drivers/media/cec/platform/seco/seco-cec.c
+@@ -649,7 +649,7 @@ static int secocec_probe(struct platform_device *pdev)
+
+ ret = secocec_ir_probe(secocec);
+ if (ret)
+- goto err_notifier;
++ goto err_unregister_adapter;
+
+ platform_set_drvdata(pdev, secocec);
+
+@@ -657,6 +657,10 @@ static int secocec_probe(struct platform_device *pdev)
+
+ return ret;
+
++err_unregister_adapter:
++ cec_notifier_cec_adap_unregister(secocec->notifier, secocec->cec_adap);
++ cec_unregister_adapter(secocec->cec_adap);
++ goto err;
+ err_notifier:
+ cec_notifier_cec_adap_unregister(secocec->notifier, secocec->cec_adap);
+ err_delete_adapter:
+diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c
+index b0523fc23506ac..0feffc7807f075 100644
+--- a/drivers/media/common/videobuf2/videobuf2-core.c
++++ b/drivers/media/common/videobuf2/videobuf2-core.c
+@@ -2991,8 +2991,8 @@ static int __vb2_cleanup_fileio(struct vb2_queue *q)
+ * @nonblock: mode selector (1 means blocking calls, 0 means nonblocking)
+ * @read: access mode selector (1 means read, 0 means write)
+ */
+-static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
+- loff_t *ppos, int nonblock, int read)
++static ssize_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
++ loff_t *ppos, int nonblock, int read)
+ {
+ struct vb2_fileio_data *fileio;
+ struct vb2_fileio_buf *buf;
+@@ -3155,15 +3155,15 @@ static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_
+ return ret;
+ }
+
+-size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
+- loff_t *ppos, int nonblocking)
++ssize_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
++ loff_t *ppos, int nonblocking)
+ {
+ return __vb2_perform_fileio(q, data, count, ppos, nonblocking, 1);
+ }
+ EXPORT_SYMBOL_GPL(vb2_read);
+
+-size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
+- loff_t *ppos, int nonblocking)
++ssize_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
++ loff_t *ppos, int nonblocking)
+ {
+ return __vb2_perform_fileio(q, (char __user *) data, count,
+ ppos, nonblocking, 0);
+diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c
+index 3b4e46dac1bf11..16f5f7e430922d 100644
+--- a/drivers/media/dvb-frontends/rtl2832.c
++++ b/drivers/media/dvb-frontends/rtl2832.c
+@@ -1115,10 +1115,10 @@ static void rtl2832_remove(struct i2c_client *client)
+
+ dev_dbg(&client->dev, "\n");
+
+- cancel_delayed_work_sync(&dev->i2c_gate_work);
+-
+ i2c_mux_del_adapters(dev->muxc);
+
++ cancel_delayed_work_sync(&dev->i2c_gate_work);
++
+ regmap_exit(dev->regmap);
+
+ kfree(dev);
+diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c
+index 023db6e793f867..91c16917d7ae5f 100644
+--- a/drivers/media/dvb-frontends/rtl2832_sdr.c
++++ b/drivers/media/dvb-frontends/rtl2832_sdr.c
+@@ -399,7 +399,8 @@ static int rtl2832_sdr_alloc_urbs(struct rtl2832_sdr_dev *dev)
+ }
+
+ /* Must be called with vb_queue_lock hold */
+-static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev)
++static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev,
++ enum vb2_buffer_state state)
+ {
+ struct platform_device *pdev = dev->pdev;
+ unsigned long flags;
+@@ -413,7 +414,7 @@ static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev)
+ buf = list_entry(dev->queued_bufs.next,
+ struct rtl2832_sdr_frame_buf, list);
+ list_del(&buf->list);
+- vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
++ vb2_buffer_done(&buf->vb.vb2_buf, state);
+ }
+ spin_unlock_irqrestore(&dev->queued_bufs_lock, flags);
+ }
+@@ -855,11 +856,15 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count)
+
+ dev_dbg(&pdev->dev, "\n");
+
+- if (!dev->udev)
++ if (!dev->udev) {
++ rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
+ return -ENODEV;
++ }
+
+- if (mutex_lock_interruptible(&dev->v4l2_lock))
++ if (mutex_lock_interruptible(&dev->v4l2_lock)) {
++ rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
+ return -ERESTARTSYS;
++ }
+
+ if (d->props->power_ctrl)
+ d->props->power_ctrl(d, 1);
+@@ -900,7 +905,11 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count)
+ if (ret)
+ goto err;
+
++ mutex_unlock(&dev->v4l2_lock);
++ return 0;
++
+ err:
++ rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
+ mutex_unlock(&dev->v4l2_lock);
+
+ return ret;
+@@ -920,7 +929,7 @@ static void rtl2832_sdr_stop_streaming(struct vb2_queue *vq)
+ rtl2832_sdr_kill_urbs(dev);
+ rtl2832_sdr_free_urbs(dev);
+ rtl2832_sdr_free_stream_bufs(dev);
+- rtl2832_sdr_cleanup_queued_bufs(dev);
++ rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_ERROR);
+ rtl2832_sdr_unset_adc(dev);
+
+ /* sleep tuner */
+diff --git a/drivers/media/i2c/alvium-csi2.c b/drivers/media/i2c/alvium-csi2.c
+index 5ddfd3dcb18876..2081714cba0ffa 100644
+--- a/drivers/media/i2c/alvium-csi2.c
++++ b/drivers/media/i2c/alvium-csi2.c
+@@ -2102,20 +2102,21 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
+ V4L2_CID_PIXEL_RATE, 0,
+ ALVIUM_DEFAULT_PIXEL_RATE_MHZ, 1,
+ ALVIUM_DEFAULT_PIXEL_RATE_MHZ);
+- ctrls->pixel_rate->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+ /* Link freq is fixed */
+ ctrls->link_freq = v4l2_ctrl_new_int_menu(hdl, ops,
+ V4L2_CID_LINK_FREQ,
+ 0, 0, &alvium->link_freq);
+- ctrls->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
++ if (ctrls->link_freq)
++ ctrls->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+ /* Auto/manual white balance */
+ if (alvium->avail_ft.auto_whiteb) {
+ ctrls->auto_wb = v4l2_ctrl_new_std(hdl, ops,
+ V4L2_CID_AUTO_WHITE_BALANCE,
+ 0, 1, 1, 1);
+- v4l2_ctrl_auto_cluster(3, &ctrls->auto_wb, 0, false);
++ if (ctrls->auto_wb)
++ v4l2_ctrl_auto_cluster(3, &ctrls->auto_wb, 0, false);
+ }
+
+ ctrls->blue_balance = v4l2_ctrl_new_std(hdl, ops,
+@@ -2124,6 +2125,7 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
+ alvium->max_bbalance,
+ alvium->inc_bbalance,
+ alvium->dft_bbalance);
++
+ ctrls->red_balance = v4l2_ctrl_new_std(hdl, ops,
+ V4L2_CID_RED_BALANCE,
+ alvium->min_rbalance,
+@@ -2138,7 +2140,9 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
+ V4L2_CID_EXPOSURE_AUTO,
+ V4L2_EXPOSURE_MANUAL, 0,
+ V4L2_EXPOSURE_AUTO);
+- v4l2_ctrl_auto_cluster(2, &ctrls->auto_exp, 1, true);
++ if (ctrls->auto_exp)
++ v4l2_ctrl_auto_cluster(2, &ctrls->auto_exp,
++ V4L2_EXPOSURE_MANUAL, true);
+ }
+
+ ctrls->exposure = v4l2_ctrl_new_std(hdl, ops,
+@@ -2147,14 +2151,16 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
+ alvium->max_exp,
+ alvium->inc_exp,
+ alvium->dft_exp);
+- ctrls->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
++ if (ctrls->exposure)
++ ctrls->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
+
+ /* Auto/manual gain */
+ if (alvium->avail_ft.auto_gain) {
+ ctrls->auto_gain = v4l2_ctrl_new_std(hdl, ops,
+ V4L2_CID_AUTOGAIN,
+ 0, 1, 1, 1);
+- v4l2_ctrl_auto_cluster(2, &ctrls->auto_gain, 0, true);
++ if (ctrls->auto_gain)
++ v4l2_ctrl_auto_cluster(2, &ctrls->auto_gain, 0, true);
+ }
+
+ if (alvium->avail_ft.gain) {
+@@ -2164,7 +2170,8 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
+ alvium->max_gain,
+ alvium->inc_gain,
+ alvium->dft_gain);
+- ctrls->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
++ if (ctrls->gain)
++ ctrls->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
+ }
+
+ if (alvium->avail_ft.sat)
+diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c
+index c8705d786cddca..5e639a99cab17b 100644
+--- a/drivers/media/pci/cx23885/cx23885-core.c
++++ b/drivers/media/pci/cx23885/cx23885-core.c
+@@ -990,8 +990,12 @@ static int cx23885_dev_setup(struct cx23885_dev *dev)
+ }
+
+ /* PCIe stuff */
+- dev->lmmio = ioremap(pci_resource_start(dev->pci, 0),
+- pci_resource_len(dev->pci, 0));
++ dev->lmmio = pci_ioremap_bar(dev->pci, 0);
++ if (!dev->lmmio) {
++ dev_err(&dev->pci->dev, "CORE %s: can't ioremap MMIO memory\n",
++ dev->name);
++ goto err_release_region;
++ }
+
+ dev->bmmio = (u8 __iomem *)dev->lmmio;
+
+@@ -1096,6 +1100,12 @@ static int cx23885_dev_setup(struct cx23885_dev *dev)
+ }
+
+ return 0;
++
++err_release_region:
++ release_mem_region(pci_resource_start(dev->pci, 0),
++ pci_resource_len(dev->pci, 0));
++ cx23885_devcount--;
++ return -ENODEV;
+ }
+
+ static void cx23885_dev_unregister(struct cx23885_dev *dev)
+diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c
+index e1185aa669f480..c1d55b98367447 100644
+--- a/drivers/media/pci/dm1105/dm1105.c
++++ b/drivers/media/pci/dm1105/dm1105.c
+@@ -1194,6 +1194,7 @@ static void dm1105_remove(struct pci_dev *pdev)
+
+ dm1105_hw_exit(dev);
+ free_irq(pdev->irq, dev);
++ destroy_workqueue(dev->wq);
+ pci_iounmap(pdev, dev->io_mem);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c b/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c
+index 1715275e6776c7..318e07592ed071 100644
+--- a/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c
++++ b/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c
+@@ -288,15 +288,27 @@ static const struct dwc_dphy_freq_range freqranges[DPHY_FREQ_RANGE_NUM] = {
+
+ static u16 get_hsfreq_by_mbps(u32 mbps)
+ {
+- unsigned int i = DPHY_FREQ_RANGE_NUM;
+-
+- while (i--) {
+- if (freqranges[i].default_mbps == mbps ||
+- (mbps >= freqranges[i].min && mbps <= freqranges[i].max))
+- return i;
++ u16 best = DPHY_FREQ_RANGE_INVALID_INDEX;
++ unsigned int i;
++
++ for (i = 0; i < DPHY_FREQ_RANGE_NUM; i++) {
++ if (mbps > freqranges[i].max)
++ continue;
++
++ if (mbps < freqranges[i].min)
++ break;
++
++ if (best == DPHY_FREQ_RANGE_INVALID_INDEX ||
++ freqranges[i].osc_freq_target >
++ freqranges[best].osc_freq_target ||
++ (freqranges[i].osc_freq_target ==
++ freqranges[best].osc_freq_target &&
++ abs((int)mbps - (int)freqranges[i].default_mbps) <
++ abs((int)mbps - (int)freqranges[best].default_mbps)))
++ best = i;
+ }
+
+- return DPHY_FREQ_RANGE_INVALID_INDEX;
++ return best;
+ }
+
+ static int ipu6_isys_dwc_phy_config(struct ipu6_isys *isys,
+diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c
+index 56b4481a40e612..75a9a951942f3c 100644
+--- a/drivers/media/pci/saa7134/saa7134-video.c
++++ b/drivers/media/pci/saa7134/saa7134-video.c
+@@ -1716,8 +1716,10 @@ int saa7134_video_init1(struct saa7134_dev *dev)
+ q->dev = &dev->pci->dev;
+ ret = vb2_queue_init(q);
+ if (ret)
+- return ret;
+- saa7134_pgtable_alloc(dev->pci, &dev->video_q.pt);
++ goto err_free_ctrl;
++ ret = saa7134_pgtable_alloc(dev->pci, &dev->video_q.pt);
++ if (ret)
++ goto err_free_ctrl;
+
+ q = &dev->vbi_vbq;
+ q->type = V4L2_BUF_TYPE_VBI_CAPTURE;
+@@ -1734,11 +1736,24 @@ int saa7134_video_init1(struct saa7134_dev *dev)
+ q->lock = &dev->lock;
+ q->dev = &dev->pci->dev;
+ ret = vb2_queue_init(q);
+- if (ret)
+- return ret;
+- saa7134_pgtable_alloc(dev->pci, &dev->vbi_q.pt);
++ if (ret) {
++ saa7134_pgtable_free(dev->pci, &dev->video_q.pt);
++ goto err_free_ctrl;
++ }
++
++ ret = saa7134_pgtable_alloc(dev->pci, &dev->vbi_q.pt);
++ if (ret) {
++ saa7134_pgtable_free(dev->pci, &dev->video_q.pt);
++ goto err_free_ctrl;
++ }
+
+ return 0;
++
++err_free_ctrl:
++ v4l2_ctrl_handler_free(&dev->ctrl_handler);
++ if (card_has_radio(dev))
++ v4l2_ctrl_handler_free(&dev->radio_ctrl_handler);
++ return ret;
+ }
+
+ void saa7134_video_fini(struct saa7134_dev *dev)
+diff --git a/drivers/media/platform/aspeed/aspeed-video.c b/drivers/media/platform/aspeed/aspeed-video.c
+index fc6050e3be0d8e..63360667cff292 100644
+--- a/drivers/media/platform/aspeed/aspeed-video.c
++++ b/drivers/media/platform/aspeed/aspeed-video.c
+@@ -2186,6 +2186,7 @@ static int aspeed_video_probe(struct platform_device *pdev)
+ rc = aspeed_video_setup_video(video);
+ if (rc) {
+ aspeed_video_free_buf(video, &video->jpeg);
++ of_reserved_mem_device_release(&pdev->dev);
+ clk_unprepare(video->vclk);
+ clk_unprepare(video->eclk);
+ return rc;
+diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+index dd34946873aceb..b7761d8a7d0ac7 100644
+--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
++++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+@@ -233,13 +233,6 @@ static int start_encode(struct vpu_instance *inst, u32 *fail_res)
+ } else {
+ dev_dbg(inst->dev->dev, "%s: wave5_vpu_enc_start_one_frame success\n",
+ __func__);
+- /*
+- * Remove the source buffer from the ready-queue now and finish
+- * it in the videobuf2 framework once the index is returned by the
+- * firmware in finish_encode
+- */
+- if (src_buf)
+- v4l2_m2m_src_buf_remove_by_idx(m2m_ctx, src_buf->vb2_buf.index);
+ }
+
+ return 0;
+@@ -266,27 +259,13 @@ static void wave5_vpu_enc_finish_encode(struct vpu_instance *inst)
+ __func__, enc_output_info.pic_type, enc_output_info.recon_frame_index,
+ enc_output_info.enc_src_idx, enc_output_info.enc_pic_byte, enc_output_info.pts);
+
+- /*
+- * The source buffer will not be found in the ready-queue as it has been
+- * dropped after sending of the encode firmware command, locate it in
+- * the videobuf2 queue directly
+- */
+ if (enc_output_info.enc_src_idx >= 0) {
+- struct vb2_buffer *vb = vb2_get_buffer(v4l2_m2m_get_src_vq(m2m_ctx),
+- enc_output_info.enc_src_idx);
+- if (vb->state != VB2_BUF_STATE_ACTIVE)
+- dev_warn(inst->dev->dev,
+- "%s: encoded buffer (%d) was not in ready queue %i.",
+- __func__, enc_output_info.enc_src_idx, vb->state);
+- else
+- src_buf = to_vb2_v4l2_buffer(vb);
+-
+- if (src_buf) {
++ src_buf = v4l2_m2m_src_buf_remove_by_idx(m2m_ctx, enc_output_info.enc_src_idx);
++ if (!src_buf) {
++ dev_warn(inst->dev->dev, "%s: no source buffer found\n", __func__);
++ } else {
+ inst->timestamp = src_buf->vb2_buf.timestamp;
+ v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE);
+- } else {
+- dev_warn(inst->dev->dev, "%s: no source buffer with index: %d found\n",
+- __func__, enc_output_info.enc_src_idx);
+ }
+ }
+
+diff --git a/drivers/media/platform/marvell/cafe-driver.c b/drivers/media/platform/marvell/cafe-driver.c
+index ef810249def61c..29cfdc3d9bd46f 100644
+--- a/drivers/media/platform/marvell/cafe-driver.c
++++ b/drivers/media/platform/marvell/cafe-driver.c
+@@ -609,6 +609,7 @@ static void cafe_pci_remove(struct pci_dev *pdev)
+ return;
+ }
+ cafe_shutdown(cam);
++ pci_disable_device(pdev);
+ kfree(cam);
+ }
+
+diff --git a/drivers/media/platform/nuvoton/npcm-video.c b/drivers/media/platform/nuvoton/npcm-video.c
+index e0dee768a3be1d..399401cfbc0fb7 100644
+--- a/drivers/media/platform/nuvoton/npcm-video.c
++++ b/drivers/media/platform/nuvoton/npcm-video.c
+@@ -1719,10 +1719,12 @@ static int npcm_video_init(struct npcm_video *video)
+ if (rc) {
+ dev_err(dev, "Failed to set DMA mask\n");
+ of_reserved_mem_device_release(dev);
++ return rc;
+ }
+
+ rc = npcm_video_ece_init(video);
+ if (rc) {
++ of_reserved_mem_device_release(dev);
+ dev_err(dev, "Failed to initialize ECE\n");
+ return rc;
+ }
+@@ -1747,42 +1749,55 @@ static int npcm_video_probe(struct platform_device *pdev)
+ regs = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(regs)) {
+ dev_err(&pdev->dev, "Failed to parse VCD reg in DTS\n");
+- return PTR_ERR(regs);
++ rc = PTR_ERR(regs);
++ goto err_free;
+ }
+
+ video->vcd_regmap = devm_regmap_init_mmio(&pdev->dev, regs,
+ &npcm_video_regmap_cfg);
+ if (IS_ERR(video->vcd_regmap)) {
+ dev_err(&pdev->dev, "Failed to initialize VCD regmap\n");
+- return PTR_ERR(video->vcd_regmap);
++ rc = PTR_ERR(video->vcd_regmap);
++ goto err_free;
+ }
+
+ video->reset = devm_reset_control_get(&pdev->dev, NULL);
+ if (IS_ERR(video->reset)) {
+ dev_err(&pdev->dev, "Failed to get VCD reset control in DTS\n");
+- return PTR_ERR(video->reset);
++ rc = PTR_ERR(video->reset);
++ goto err_free;
+ }
+
+ video->gcr_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+ "nuvoton,sysgcr");
+- if (IS_ERR(video->gcr_regmap))
+- return PTR_ERR(video->gcr_regmap);
++ if (IS_ERR(video->gcr_regmap)) {
++ rc = PTR_ERR(video->gcr_regmap);
++ goto err_free;
++ }
+
+ video->gfx_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+ "nuvoton,sysgfxi");
+- if (IS_ERR(video->gfx_regmap))
+- return PTR_ERR(video->gfx_regmap);
++ if (IS_ERR(video->gfx_regmap)) {
++ rc = PTR_ERR(video->gfx_regmap);
++ goto err_free;
++ }
+
+ rc = npcm_video_init(video);
+ if (rc)
+- return rc;
++ goto err_free;
+
+ rc = npcm_video_setup_video(video);
+ if (rc)
+- return rc;
++ goto err_release_mem;
+
+ dev_info(video->dev, "NPCM video driver probed\n");
+ return 0;
++
++err_release_mem:
++ of_reserved_mem_device_release(&pdev->dev);
++err_free:
++ kfree(video);
++ return rc;
+ }
+
+ static void npcm_video_remove(struct platform_device *pdev)
+@@ -1797,6 +1812,7 @@ static void npcm_video_remove(struct platform_device *pdev)
+ v4l2_device_unregister(v4l2_dev);
+ if (video->ece.enable)
+ npcm_video_ece_stop(video);
++ kfree(video);
+ of_reserved_mem_device_release(dev);
+ }
+
+diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
+index a161dc32689542..5b991c9ff6fd93 100644
+--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
++++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
+@@ -501,6 +501,8 @@ static int mxc_isi_probe(struct platform_device *pdev)
+ return 0;
+
+ err_xbar:
++ while (i--)
++ mxc_isi_pipe_cleanup(&isi->pipes[i]);
+ mxc_isi_crossbar_cleanup(&isi->crossbar);
+
+ return ret;
+diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
+index 5a4676d5207935..467ce2b877725f 100644
+--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
++++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
+@@ -11,6 +11,7 @@
+ #define __MXC_ISI_CORE_H__
+
+ #include <linux/list.h>
++#include <linux/math.h>
+ #include <linux/mutex.h>
+ #include <linux/spinlock.h>
+ #include <linux/types.h>
+@@ -403,4 +404,19 @@ static inline void mxc_isi_debug_cleanup(struct mxc_isi_dev *isi)
+ }
+ #endif
+
++/*
++ * ISI scaling engine works in two parts: it performs pre-decimation of
++ * the image followed by bilinear filtering to achieve the desired
++ * downscaling factor.
++ *
++ * The decimation filter provides a maximum downscaling factor of 8, and
++ * the subsequent bilinear filter provides a maximum downscaling factor
++ * of 2. Combined, the maximum scaling factor can be up to 16.
++ */
++static inline unsigned int
++mxc_isi_clamp_downscale_16(unsigned int val, unsigned int max_val)
++{
++ return clamp(val, max(1U, DIV_ROUND_UP(max_val, 16)), max_val);
++}
++
+ #endif /* __MXC_ISI_CORE_H__ */
+diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
+index 93a55c97cd1738..880275de003898 100644
+--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
++++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
+@@ -494,6 +494,7 @@ err_free:
+
+ void mxc_isi_crossbar_cleanup(struct mxc_isi_crossbar *xbar)
+ {
++ v4l2_subdev_cleanup(&xbar->sd);
+ media_entity_cleanup(&xbar->sd.entity);
+ kfree(xbar->pads);
+ kfree(xbar->inputs);
+diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
+index 9225a7ac1c3ee7..0bfd9b8f5f1088 100644
+--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
++++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
+@@ -112,7 +112,14 @@ static u32 mxc_isi_channel_scaling_ratio(unsigned int from, unsigned int to,
+ else
+ *dec = 8;
+
+- return min_t(u32, from * 0x1000 / (to * *dec), ISI_DOWNSCALE_THRESHOLD);
++ /*
++ * The ISI rounds output dimensions up to the next integer (i.MX93 RM
++ * section 57.7.8). Calculate the scale factor such that the theoretical
++ * output (input / scale_factor) rounds up to exactly the desired
++ * output.
++ */
++ return min_t(u32, DIV_ROUND_UP(from * 0x1000, to * *dec),
++ ISI_DOWNSCALE_THRESHOLD);
+ }
+
+ static void mxc_isi_channel_set_scaling(struct mxc_isi_pipe *pipe,
+diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
+index 81223d28ee56e8..cb18ce84466d8b 100644
+--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
++++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
+@@ -504,9 +504,14 @@ __mxc_isi_m2m_try_fmt_vid(struct mxc_isi_m2m_ctx *ctx,
+ const enum mxc_isi_video_type type)
+ {
+ if (type == MXC_ISI_VIDEO_M2M_CAP) {
+- /* Downscaling only */
+- pix->width = min(pix->width, ctx->queues.out.format.width);
+- pix->height = min(pix->height, ctx->queues.out.format.height);
++ const struct v4l2_pix_format_mplane *format =
++ &ctx->queues.out.format;
++
++ /* Downscaling only, by up to 16. */
++ pix->width = mxc_isi_clamp_downscale_16(pix->width,
++ format->width);
++ pix->height = mxc_isi_clamp_downscale_16(pix->height,
++ format->height);
+ }
+
+ return mxc_isi_format_try(ctx->m2m->pipe, pix, type);
+diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
+index a41c51dd9ce0f2..2d0843c86534cc 100644
+--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
++++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
+@@ -641,16 +641,19 @@ static int mxc_isi_pipe_set_selection(struct v4l2_subdev *sd,
+ /* Composing is supported on the sink only. */
+ return -EINVAL;
+
+- /* The sink crop is bound by the sink format downscaling only). */
++ /*
++ * The ISI supports downscaling only, with a factor up to 16.
++ * Clamp the compose rectangle size accordingly.
++ */
+ format = mxc_isi_pipe_get_pad_format(pipe, state,
+ MXC_ISI_PIPE_PAD_SINK);
+
+ sel->r.left = 0;
+ sel->r.top = 0;
+- sel->r.width = clamp(sel->r.width, MXC_ISI_MIN_WIDTH,
+- format->width);
+- sel->r.height = clamp(sel->r.height, MXC_ISI_MIN_HEIGHT,
+- format->height);
++ sel->r.width = mxc_isi_clamp_downscale_16(sel->r.width,
++ format->width);
++ sel->r.height = mxc_isi_clamp_downscale_16(sel->r.height,
++ format->height);
+
+ rect = mxc_isi_pipe_get_pad_compose(pipe, state,
+ MXC_ISI_PIPE_PAD_SINK);
+@@ -796,18 +799,20 @@ int mxc_isi_pipe_init(struct mxc_isi_dev *isi, unsigned int id)
+ irq = platform_get_irq(to_platform_device(isi->dev), id);
+ if (irq < 0) {
+ ret = irq;
+- goto error;
++ goto error_subdev;
+ }
+
+ ret = devm_request_irq(isi->dev, irq, mxc_isi_pipe_irq_handler,
+ 0, dev_name(isi->dev), pipe);
+ if (ret < 0) {
+ dev_err(isi->dev, "failed to request IRQ (%d)\n", ret);
+- goto error;
++ goto error_subdev;
+ }
+
+ return 0;
+
++error_subdev:
++ v4l2_subdev_cleanup(sd);
+ error:
+ media_entity_cleanup(&sd->entity);
+ mutex_destroy(&pipe->lock);
+@@ -819,6 +824,7 @@ void mxc_isi_pipe_cleanup(struct mxc_isi_pipe *pipe)
+ {
+ struct v4l2_subdev *sd = &pipe->sd;
+
++ v4l2_subdev_cleanup(sd);
+ media_entity_cleanup(&sd->entity);
+ mutex_destroy(&pipe->lock);
+ }
+diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen2.c b/drivers/media/platform/qcom/camss/camss-csid-gen2.c
+index e1c757933e2733..b366955ea43446 100644
+--- a/drivers/media/platform/qcom/camss/camss-csid-gen2.c
++++ b/drivers/media/platform/qcom/camss/camss-csid-gen2.c
+@@ -208,10 +208,10 @@ static void __csid_ctrl_rdi(struct csid_device *csid, int enable, u8 rdi)
+ writel_relaxed(val, csid->base + CSID_RDI_CTRL(rdi));
+ }
+
+-static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 vc)
++static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 port, u8 vc)
+ {
+ struct csid_testgen_config *tg = &csid->testgen;
+- struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
++ struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port];
+ const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats,
+ csid->res->formats->nformats,
+ input_format->code);
+@@ -258,10 +258,10 @@ static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 vc)
+ writel_relaxed(val, csid->base + CSID_TPG_CTRL);
+ }
+
+-static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc)
++static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc)
+ {
+ /* Source pads matching RDI channels on hardware. Pad 1 -> RDI0, Pad 2 -> RDI1, etc. */
+- struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
++ struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port];
+ const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats,
+ csid->res->formats->nformats,
+ input_format->code);
+@@ -272,14 +272,14 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
+ * the four least significant bits of the five bit VC
+ * bitfield to generate an internal CID value.
+ *
+- * CSID_RDI_CFG0(vc)
++ * CSID_RDI_CFG0(port)
+ * DT_ID : 28:27
+ * VC : 26:22
+ * DT : 21:16
+ *
+ * CID : VC 3:0 << 2 | DT_ID 1:0
+ */
+- u8 dt_id = vc & 0x03;
++ u8 dt_id = port & 0x03;
+
+ val = 1 << RDI_CFG0_BYTE_CNTR_EN;
+ val |= 1 << RDI_CFG0_FORMAT_MEASURE_EN;
+@@ -289,56 +289,57 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
+ val |= format->data_type << RDI_CFG0_DATA_TYPE;
+ val |= vc << RDI_CFG0_VIRTUAL_CHANNEL;
+ val |= dt_id << RDI_CFG0_DT_ID;
+- writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_CFG0(port));
+
+ /* CSID_TIMESTAMP_STB_POST_IRQ */
+ val = 2 << RDI_CFG1_TIMESTAMP_STB_SEL;
+- writel_relaxed(val, csid->base + CSID_RDI_CFG1(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_CFG1(port));
+
+ val = 1;
+- writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(port));
+
+ val = 0;
+- writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(port));
+
+ val = 1;
+- writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(port));
+
+ val = 0;
+- writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(port));
+
+ val = 1;
+- writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(port));
+
+ val = 0;
+- writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(port));
+
+ val = 1;
+- writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(port));
+
+ val = 0;
+- writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(port));
+
+ val = 0;
+- writel_relaxed(val, csid->base + CSID_RDI_CTRL(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_CTRL(port));
+
+- val = readl_relaxed(csid->base + CSID_RDI_CFG0(vc));
++ val = readl_relaxed(csid->base + CSID_RDI_CFG0(port));
+ val |= enable << RDI_CFG0_ENABLE;
+- writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc));
++ writel_relaxed(val, csid->base + CSID_RDI_CFG0(port));
+ }
+
+ static void csid_configure_stream(struct csid_device *csid, u8 enable)
+ {
+ struct csid_testgen_config *tg = &csid->testgen;
+ u8 i;
+- /* Loop through all enabled VCs and configure stream for each */
++
++ /* Loop through all enabled ports and configure a stream for each */
+ for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++)
+ if (csid->phy.en_vc & BIT(i)) {
+ if (tg->enabled)
+- __csid_configure_testgen(csid, enable, i);
++ __csid_configure_testgen(csid, enable, i, 0);
+
+- __csid_configure_rdi_stream(csid, enable, i);
+- __csid_configure_rx(csid, &csid->phy, i);
++ __csid_configure_rdi_stream(csid, enable, i, 0);
++ __csid_configure_rx(csid, &csid->phy, 0);
+ __csid_ctrl_rdi(csid, enable, i);
+ }
+ }
+diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c
+index ff3331af940689..a210b5a2f9e18b 100644
+--- a/drivers/media/platform/st/stm32/stm32-dcmi.c
++++ b/drivers/media/platform/st/stm32/stm32-dcmi.c
+@@ -2069,6 +2069,7 @@ static int dcmi_probe(struct platform_device *pdev)
+ return 0;
+
+ err_cleanup:
++ v4l2_async_nf_unregister(&dcmi->notifier);
+ v4l2_async_nf_cleanup(&dcmi->notifier);
+ err_media_entity_cleanup:
+ media_entity_cleanup(&dcmi->vdev->entity);
+diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c
+index d1371e13011374..5c04b5dd0a20e8 100644
+--- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c
++++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c
+@@ -234,8 +234,10 @@ static int sun4i_csi_start_streaming(struct vb2_queue *vq, unsigned int count)
+ int ret;
+
+ csi_fmt = sun4i_csi_find_format(&csi->fmt.pixelformat, NULL);
+- if (!csi_fmt)
+- return -EINVAL;
++ if (!csi_fmt) {
++ ret = -EINVAL;
++ goto err_clear_dma_queue;
++ }
+
+ dev_dbg(csi->dev, "Starting capture\n");
+
+diff --git a/drivers/media/platform/ti/davinci/vpif_capture.c b/drivers/media/platform/ti/davinci/vpif_capture.c
+index d8beebd1b090a2..fe44d3abe18e75 100644
+--- a/drivers/media/platform/ti/davinci/vpif_capture.c
++++ b/drivers/media/platform/ti/davinci/vpif_capture.c
+@@ -1501,7 +1501,7 @@ vpif_capture_get_pdata(struct platform_device *pdev,
+ * video ports & endpoints data.
+ */
+ if (pdev->dev.parent && pdev->dev.parent->of_node)
+- pdev->dev.of_node = pdev->dev.parent->of_node;
++ device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent);
+ if (!IS_ENABLED(CONFIG_OF) || !pdev->dev.of_node)
+ return pdev->dev.platform_data;
+
+diff --git a/drivers/media/platform/ti/vpe/vpe.c b/drivers/media/platform/ti/vpe/vpe.c
+index 6848cbc82f5288..a2d938e8050d65 100644
+--- a/drivers/media/platform/ti/vpe/vpe.c
++++ b/drivers/media/platform/ti/vpe/vpe.c
+@@ -2545,7 +2545,8 @@ static int vpe_probe(struct platform_device *pdev)
+ "vpe_top");
+ if (!dev->res) {
+ dev_err(&pdev->dev, "missing 'vpe_top' resources data\n");
+- return -ENODEV;
++ ret = -ENODEV;
++ goto v4l2_dev_unreg;
+ }
+
+ /*
+diff --git a/drivers/media/radio/radio-si476x.c b/drivers/media/radio/radio-si476x.c
+index b2c5809a8bc71f..1a713f51ffb5df 100644
+--- a/drivers/media/radio/radio-si476x.c
++++ b/drivers/media/radio/radio-si476x.c
+@@ -1493,6 +1493,7 @@ static int si476x_radio_probe(struct platform_device *pdev)
+ return 0;
+ exit:
+ v4l2_ctrl_handler_free(radio->videodev.ctrl_handler);
++ v4l2_device_unregister(&radio->v4l2dev);
+ return rval;
+ }
+
+diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+index 743478bf96f231..cdc5d6b08075d5 100644
+--- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c
++++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+@@ -594,8 +594,10 @@ static int __init vidtv_bridge_init(void)
+ int ret;
+
+ ret = platform_device_register(&vidtv_bridge_dev);
+- if (ret)
++ if (ret) {
++ platform_device_put(&vidtv_bridge_dev);
+ return ret;
++ }
+
+ ret = platform_driver_register(&vidtv_bridge_driver);
+ if (ret)
+diff --git a/drivers/media/test-drivers/vimc/vimc-core.c b/drivers/media/test-drivers/vimc/vimc-core.c
+index 2083c60e34d6a6..8f10faa0530750 100644
+--- a/drivers/media/test-drivers/vimc/vimc-core.c
++++ b/drivers/media/test-drivers/vimc/vimc-core.c
+@@ -424,6 +424,7 @@ static int __init vimc_init(void)
+ if (ret) {
+ dev_err(&vimc_pdev.dev,
+ "platform device registration failed (err=%d)\n", ret);
++ platform_device_put(&vimc_pdev);
+ return ret;
+ }
+
+diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c
+index 4f330f4fc6be9e..cf9f37bc0c08a2 100644
+--- a/drivers/media/test-drivers/vivid/vivid-core.c
++++ b/drivers/media/test-drivers/vivid/vivid-core.c
+@@ -2291,8 +2291,10 @@ static int __init vivid_init(void)
+ }
+ }
+ ret = platform_device_register(&vivid_pdev);
+- if (ret)
++ if (ret) {
++ platform_device_put(&vivid_pdev);
+ goto free_output_strings;
++ }
+ ret = platform_driver_register(&vivid_pdrv);
+ if (ret)
+ goto unreg_device;
+@@ -2313,7 +2315,7 @@ static int __init vivid_init(void)
+ destroy_hdmi_wq:
+ destroy_workqueue(update_hdmi_ctrls_workqueue);
+ unreg_driver:
+- platform_driver_register(&vivid_pdrv);
++ platform_driver_unregister(&vivid_pdrv);
+ unreg_device:
+ platform_device_unregister(&vivid_pdev);
+ free_output_strings:
+diff --git a/drivers/media/test-drivers/vivid/vivid-ctrls.c b/drivers/media/test-drivers/vivid/vivid-ctrls.c
+index 3fb4e08ac725ba..0655609a7f881f 100644
+--- a/drivers/media/test-drivers/vivid/vivid-ctrls.c
++++ b/drivers/media/test-drivers/vivid/vivid-ctrls.c
+@@ -576,17 +576,24 @@ static int vivid_vid_cap_s_ctrl(struct v4l2_ctrl *ctrl)
+ break;
+ case VIVID_CID_REDUCED_FPS:
+ dev->reduced_fps = ctrl->val;
+- vivid_update_format_cap(dev, true);
++ if (dev->input_type[dev->input] == HDMI)
++ vivid_update_reduced_fps(dev);
+ break;
+ case VIVID_CID_HAS_CROP_CAP:
++ if (vb2_is_busy(&dev->vb_vid_cap_q))
++ return -EBUSY;
+ dev->has_crop_cap = ctrl->val;
+ vivid_update_format_cap(dev, true);
+ break;
+ case VIVID_CID_HAS_COMPOSE_CAP:
++ if (vb2_is_busy(&dev->vb_vid_cap_q))
++ return -EBUSY;
+ dev->has_compose_cap = ctrl->val;
+ vivid_update_format_cap(dev, true);
+ break;
+ case VIVID_CID_HAS_SCALER_CAP:
++ if (vb2_is_busy(&dev->vb_vid_cap_q))
++ return -EBUSY;
+ dev->has_scaler_cap = ctrl->val;
+ vivid_update_format_cap(dev, true);
+ break;
+@@ -1083,14 +1090,20 @@ static int vivid_vid_out_s_ctrl(struct v4l2_ctrl *ctrl)
+
+ switch (ctrl->id) {
+ case VIVID_CID_HAS_CROP_OUT:
++ if (vb2_is_busy(&dev->vb_vid_out_q))
++ return -EBUSY;
+ dev->has_crop_out = ctrl->val;
+ vivid_update_format_out(dev);
+ break;
+ case VIVID_CID_HAS_COMPOSE_OUT:
++ if (vb2_is_busy(&dev->vb_vid_out_q))
++ return -EBUSY;
+ dev->has_compose_out = ctrl->val;
+ vivid_update_format_out(dev);
+ break;
+ case VIVID_CID_HAS_SCALER_OUT:
++ if (vb2_is_busy(&dev->vb_vid_out_q))
++ return -EBUSY;
+ dev->has_scaler_out = ctrl->val;
+ vivid_update_format_out(dev);
+ break;
+diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
+index cc84d2671d8407..327a7f8cd00d9f 100644
+--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c
++++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
+@@ -361,6 +361,24 @@ static enum tpg_pixel_aspect vivid_get_pixel_aspect(const struct vivid_dev *dev)
+ return TPG_PIXEL_ASPECT_SQUARE;
+ }
+
++void vivid_update_reduced_fps(struct vivid_dev *dev)
++{
++ struct v4l2_bt_timings *bt = &dev->dv_timings_cap[dev->input].bt;
++ unsigned int size = V4L2_DV_BT_FRAME_WIDTH(bt) * V4L2_DV_BT_FRAME_HEIGHT(bt);
++ u64 pixelclock;
++
++ if (dev->reduced_fps && can_reduce_fps(bt)) {
++ pixelclock = div_u64(bt->pixelclock * 1000, 1001);
++ bt->flags |= V4L2_DV_FL_REDUCED_FPS;
++ } else {
++ pixelclock = bt->pixelclock;
++ bt->flags &= ~V4L2_DV_FL_REDUCED_FPS;
++ }
++ dev->timeperframe_vid_cap = (struct v4l2_fract) {
++ size / 100, (u32)pixelclock / 100
++ };
++}
++
+ /*
+ * Called whenever the format has to be reset which can occur when
+ * changing inputs, standard, timings, etc.
+@@ -369,8 +387,12 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls)
+ {
+ struct v4l2_bt_timings *bt = &dev->dv_timings_cap[dev->input].bt;
+ u32 dims[V4L2_CTRL_MAX_DIMS] = {};
+- unsigned size;
+- u64 pixelclock;
++
++ /*
++ * This resets the format, so must never be called while vb2_is_busy().
++ */
++ if (WARN_ON(vb2_is_busy(&dev->vb_vid_cap_q)))
++ return;
+
+ switch (dev->input_type[dev->input]) {
+ case WEBCAM:
+@@ -399,17 +421,7 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls)
+ case HDMI:
+ dev->src_rect.width = bt->width;
+ dev->src_rect.height = bt->height;
+- size = V4L2_DV_BT_FRAME_WIDTH(bt) * V4L2_DV_BT_FRAME_HEIGHT(bt);
+- if (dev->reduced_fps && can_reduce_fps(bt)) {
+- pixelclock = div_u64(bt->pixelclock * 1000, 1001);
+- bt->flags |= V4L2_DV_FL_REDUCED_FPS;
+- } else {
+- pixelclock = bt->pixelclock;
+- bt->flags &= ~V4L2_DV_FL_REDUCED_FPS;
+- }
+- dev->timeperframe_vid_cap = (struct v4l2_fract) {
+- size / 100, (u32)pixelclock / 100
+- };
++ vivid_update_reduced_fps(dev);
+ if (bt->interlaced)
+ dev->field_cap = V4L2_FIELD_ALTERNATE;
+ else
+diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.h b/drivers/media/test-drivers/vivid/vivid-vid-cap.h
+index 7a8daf0af2ca5a..1556c1a282ff78 100644
+--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.h
++++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.h
+@@ -9,6 +9,7 @@
+ #define _VIVID_VID_CAP_H_
+
+ void vivid_update_quality(struct vivid_dev *dev);
++void vivid_update_reduced_fps(struct vivid_dev *dev);
+ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls);
+ void vivid_update_outputs(struct vivid_dev *dev);
+ void vivid_update_connected_outputs(struct vivid_dev *dev);
+diff --git a/drivers/media/test-drivers/vivid/vivid-vid-out.c b/drivers/media/test-drivers/vivid/vivid-vid-out.c
+index 60327f3612af40..f40fb13f5a8e22 100644
+--- a/drivers/media/test-drivers/vivid/vivid-vid-out.c
++++ b/drivers/media/test-drivers/vivid/vivid-vid-out.c
+@@ -215,6 +215,12 @@ void vivid_update_format_out(struct vivid_dev *dev)
+ unsigned size, p;
+ u64 pixelclock;
+
++ /*
++ * This resets the format, so must never be called while vb2_is_busy().
++ */
++ if (WARN_ON(vb2_is_busy(&dev->vb_vid_out_q)))
++ return;
++
+ switch (dev->output_type[dev->output]) {
+ case SVID:
+ default:
+diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c
+index e24e655fb1dbfb..8cdaf8dd4f887f 100644
+--- a/drivers/media/usb/airspy/airspy.c
++++ b/drivers/media/usb/airspy/airspy.c
+@@ -522,11 +522,13 @@ static int airspy_start_streaming(struct vb2_queue *vq, unsigned int count)
+
+ dev_dbg(s->dev, "\n");
+
+- if (!s->udev)
+- return -ENODEV;
+-
+ mutex_lock(&s->v4l2_lock);
+
++ if (!s->udev) {
++ ret = -ENODEV;
++ goto err_clear_bit;
++ }
++
+ s->sequence = 0;
+
+ set_bit(POWER_ON, &s->flags);
+diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c
+index bda729b42d05fe..64c3cb8fc2bf00 100644
+--- a/drivers/media/usb/cx231xx/cx231xx-cards.c
++++ b/drivers/media/usb/cx231xx/cx231xx-cards.c
+@@ -1577,7 +1577,8 @@ static int cx231xx_init_v4l2(struct cx231xx *dev,
+ dev->video_mode.end_point_addr,
+ dev->video_mode.num_alt);
+
+- dev->video_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->video_mode.num_alt, GFP_KERNEL);
++ dev->video_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
++ dev->video_mode.num_alt, GFP_KERNEL);
+ if (dev->video_mode.alt_max_pkt_size == NULL)
+ return -ENOMEM;
+
+@@ -1618,7 +1619,8 @@ static int cx231xx_init_v4l2(struct cx231xx *dev,
+ dev->vbi_mode.num_alt);
+
+ /* compute alternate max packet sizes for vbi */
+- dev->vbi_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->vbi_mode.num_alt, GFP_KERNEL);
++ dev->vbi_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
++ dev->vbi_mode.num_alt, GFP_KERNEL);
+ if (dev->vbi_mode.alt_max_pkt_size == NULL)
+ return -ENOMEM;
+
+@@ -1660,7 +1662,9 @@ static int cx231xx_init_v4l2(struct cx231xx *dev,
+ "sliced CC EndPoint Addr 0x%x, Alternate settings: %i\n",
+ dev->sliced_cc_mode.end_point_addr,
+ dev->sliced_cc_mode.num_alt);
+- dev->sliced_cc_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->sliced_cc_mode.num_alt, GFP_KERNEL);
++ dev->sliced_cc_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
++ dev->sliced_cc_mode.num_alt,
++ GFP_KERNEL);
+ if (dev->sliced_cc_mode.alt_max_pkt_size == NULL)
+ return -ENOMEM;
+
+@@ -1724,7 +1728,7 @@ static int cx231xx_usb_probe(struct usb_interface *interface,
+ udev = usb_get_dev(interface_to_usbdev(interface));
+
+ /* allocate memory for our device state and initialize it */
+- dev = devm_kzalloc(&udev->dev, sizeof(*dev), GFP_KERNEL);
++ dev = devm_kzalloc(&interface->dev, sizeof(*dev), GFP_KERNEL);
+ if (dev == NULL) {
+ retval = -ENOMEM;
+ goto err_if;
+@@ -1854,7 +1858,9 @@ static int cx231xx_usb_probe(struct usb_interface *interface,
+ dev->ts1_mode.end_point_addr,
+ dev->ts1_mode.num_alt);
+
+- dev->ts1_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->ts1_mode.num_alt, GFP_KERNEL);
++ dev->ts1_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
++ dev->ts1_mode.num_alt,
++ GFP_KERNEL);
+ if (dev->ts1_mode.alt_max_pkt_size == NULL) {
+ retval = -ENOMEM;
+ goto err_video_alt;
+diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c
+index 5138486abfa0d8..9732a20a749e64 100644
+--- a/drivers/media/usb/msi2500/msi2500.c
++++ b/drivers/media/usb/msi2500/msi2500.c
+@@ -541,7 +541,8 @@ static int msi2500_isoc_init(struct msi2500_dev *dev)
+ }
+
+ /* Must be called with vb_queue_lock hold */
+-static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev)
++static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev,
++ enum vb2_buffer_state state)
+ {
+ unsigned long flags;
+
+@@ -554,7 +555,7 @@ static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev)
+ buf = list_entry(dev->queued_bufs.next,
+ struct msi2500_frame_buf, list);
+ list_del(&buf->list);
+- vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
++ vb2_buffer_done(&buf->vb.vb2_buf, state);
+ }
+ spin_unlock_irqrestore(&dev->queued_bufs_lock, flags);
+ }
+@@ -830,25 +831,40 @@ static int msi2500_start_streaming(struct vb2_queue *vq, unsigned int count)
+
+ dev_dbg(dev->dev, "\n");
+
+- if (!dev->udev)
+- return -ENODEV;
++ if (!dev->udev) {
++ ret = -ENODEV;
++ goto err_cleanup;
++ }
+
+- if (mutex_lock_interruptible(&dev->v4l2_lock))
+- return -ERESTARTSYS;
++ if (mutex_lock_interruptible(&dev->v4l2_lock)) {
++ ret = -ERESTARTSYS;
++ goto err_cleanup;
++ }
+
+ /* wake-up tuner */
+ v4l2_subdev_call(dev->v4l2_subdev, core, s_power, 1);
+
+ ret = msi2500_set_usb_adc(dev);
++ if (ret)
++ goto err_unlock_cleanup;
+
+ ret = msi2500_isoc_init(dev);
+ if (ret)
+- msi2500_cleanup_queued_bufs(dev);
++ goto err_unlock_cleanup;
+
+ ret = msi2500_ctrl_msg(dev, CMD_START_STREAMING, 0);
++ if (ret)
++ goto err_isoc_cleanup;
+
+ mutex_unlock(&dev->v4l2_lock);
++ return 0;
+
++err_isoc_cleanup:
++ msi2500_isoc_cleanup(dev);
++err_unlock_cleanup:
++ mutex_unlock(&dev->v4l2_lock);
++err_cleanup:
++ msi2500_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
+ return ret;
+ }
+
+@@ -863,7 +879,7 @@ static void msi2500_stop_streaming(struct vb2_queue *vq)
+ if (dev->udev)
+ msi2500_isoc_cleanup(dev);
+
+- msi2500_cleanup_queued_bufs(dev);
++ msi2500_cleanup_queued_bufs(dev, VB2_BUF_STATE_ERROR);
+
+ /* according to tests, at least 700us delay is required */
+ msleep(20);
+diff --git a/drivers/media/usb/pwc/pwc-if.c b/drivers/media/usb/pwc/pwc-if.c
+index e342199711d397..0ab96c4c7eccd4 100644
+--- a/drivers/media/usb/pwc/pwc-if.c
++++ b/drivers/media/usb/pwc/pwc-if.c
+@@ -711,11 +711,15 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count)
+ struct pwc_device *pdev = vb2_get_drv_priv(vq);
+ int r;
+
+- if (!pdev->udev)
++ if (!pdev->udev) {
++ pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED);
+ return -ENODEV;
++ }
+
+- if (mutex_lock_interruptible(&pdev->v4l2_lock))
++ if (mutex_lock_interruptible(&pdev->v4l2_lock)) {
++ pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED);
+ return -ERESTARTSYS;
++ }
+ /* Turn on camera and set LEDS on */
+ pwc_camera_power(pdev, 1);
+ pwc_set_leds(pdev, leds[0], leds[1]);
+@@ -727,6 +731,11 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count)
+ pwc_camera_power(pdev, 0);
+ /* And cleanup any queued bufs!! */
+ pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED);
++ if (pdev->fill_buf) {
++ vb2_buffer_done(&pdev->fill_buf->vb.vb2_buf,
++ VB2_BUF_STATE_QUEUED);
++ pdev->fill_buf = NULL;
++ }
+ }
+ mutex_unlock(&pdev->v4l2_lock);
+
+diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c
+index 4cc32685124c3a..1a47cf18b56fc9 100644
+--- a/drivers/media/v4l2-core/v4l2-ctrls-core.c
++++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c
+@@ -812,6 +812,7 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
+ struct v4l2_ctrl_h264_decode_params *p_h264_dec_params;
+ struct v4l2_ctrl_hevc_sps *p_hevc_sps;
+ struct v4l2_ctrl_hevc_pps *p_hevc_pps;
++ struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params;
+ struct v4l2_ctrl_hdr10_mastering_display *p_hdr10_mastering;
+ struct v4l2_ctrl_hevc_decode_params *p_hevc_decode_params;
+ struct v4l2_area *area;
+@@ -1100,6 +1101,18 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
+ break;
+
+ case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS:
++ p_hevc_slice_params = p;
++
++ if (p_hevc_slice_params->num_ref_idx_l0_active_minus1 >=
++ V4L2_HEVC_DPB_ENTRIES_NUM_MAX)
++ return -EINVAL;
++
++ if (p_hevc_slice_params->slice_type != V4L2_HEVC_SLICE_TYPE_B)
++ break;
++
++ if (p_hevc_slice_params->num_ref_idx_l1_active_minus1 >=
++ V4L2_HEVC_DPB_ENTRIES_NUM_MAX)
++ return -EINVAL;
+ break;
+
+ case V4L2_CTRL_TYPE_HDR10_CLL_INFO:
+diff --git a/drivers/media/v4l2-core/v4l2-ctrls-request.c b/drivers/media/v4l2-core/v4l2-ctrls-request.c
+index c637049d7a2b3f..dd6ca914349035 100644
+--- a/drivers/media/v4l2-core/v4l2-ctrls-request.c
++++ b/drivers/media/v4l2-core/v4l2-ctrls-request.c
+@@ -348,13 +348,12 @@ void v4l2_ctrl_request_complete(struct media_request *req,
+ ret = v4l2_ctrl_handler_init(hdl, (main_hdl->nr_of_buckets - 1) * 8);
+ if (!ret)
+ ret = v4l2_ctrl_request_bind(req, hdl, main_hdl);
+- if (ret) {
+- v4l2_ctrl_handler_free(hdl);
+- kfree(hdl);
+- return;
+- }
++ if (ret)
++ goto error;
+ hdl->request_is_queued = true;
+ obj = media_request_object_find(req, &req_ops, main_hdl);
++ if (!obj)
++ goto error;
+ }
+ hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj);
+
+@@ -389,6 +388,11 @@ void v4l2_ctrl_request_complete(struct media_request *req,
+ mutex_unlock(main_hdl->lock);
+ media_request_object_complete(obj);
+ media_request_object_put(obj);
++ return;
++
++error:
++ v4l2_ctrl_handler_free(hdl);
++ kfree(hdl);
+ }
+ EXPORT_SYMBOL(v4l2_ctrl_request_complete);
+
+diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c
+index f19c8adf2c61d9..354162eb9325c8 100644
+--- a/drivers/media/v4l2-core/v4l2-fwnode.c
++++ b/drivers/media/v4l2-core/v4l2-fwnode.c
+@@ -1205,7 +1205,7 @@ v4l2_async_nf_parse_fwnode_sensor(struct device *dev,
+ return 0;
+ }
+
+-int v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd)
++int __v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd, struct module *module)
+ {
+ struct v4l2_async_notifier *notifier;
+ int ret;
+@@ -1231,7 +1231,7 @@ int v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd)
+ if (ret < 0)
+ goto out_cleanup;
+
+- ret = v4l2_async_register_subdev(sd);
++ ret = __v4l2_async_register_subdev(sd, module);
+ if (ret < 0)
+ goto out_unregister;
+
+@@ -1249,7 +1249,7 @@ out_cleanup:
+
+ return ret;
+ }
+-EXPORT_SYMBOL_GPL(v4l2_async_register_subdev_sensor);
++EXPORT_SYMBOL_GPL(__v4l2_async_register_subdev_sensor);
+
+ MODULE_DESCRIPTION("V4L2 fwnode binding parsing library");
+ MODULE_LICENSE("GPL");
+diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
+index 1193852bad599b..01c1fe37095cf7 100644
+--- a/drivers/media/v4l2-core/v4l2-subdev.c
++++ b/drivers/media/v4l2-core/v4l2-subdev.c
+@@ -2438,6 +2438,10 @@ int v4l2_subdev_s_stream_helper(struct v4l2_subdev *sd, int enable)
+ u64 source_mask = 0;
+ int pad_index = -1;
+
++ if (WARN_ON(!v4l2_subdev_has_op(sd, pad, enable_streams) ||
++ !v4l2_subdev_has_op(sd, pad, disable_streams)))
++ return -ENOIOCTLCMD;
++
+ /*
+ * Find the source pad. This helper is meant for subdevs that have a
+ * single source pad, so failures shouldn't happen, but catch them
+diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c
+index 04f9a4b79d8508..9e22c9bdea5ea5 100644
+--- a/drivers/misc/mei/bus.c
++++ b/drivers/misc/mei/bus.c
+@@ -4,6 +4,7 @@
+ * Intel Management Engine Interface (Intel MEI) Linux driver
+ */
+
++#include <linux/cleanup.h>
+ #include <linux/module.h>
+ #include <linux/device.h>
+ #include <linux/kernel.h>
+@@ -1353,15 +1354,16 @@ static void mei_dev_bus_put(struct mei_device *bus)
+ static void mei_cl_bus_dev_release(struct device *dev)
+ {
+ struct mei_cl_device *cldev = to_mei_cl_device(dev);
+- struct mei_device *mdev = cldev->cl->dev;
++ struct mei_device *bus = cldev->bus;
+ struct mei_cl *cl;
+
+- mei_cl_flush_queues(cldev->cl, NULL);
+- mei_me_cl_put(cldev->me_cl);
+- mei_dev_bus_put(cldev->bus);
+-
+- list_for_each_entry(cl, &mdev->file_list, link)
+- WARN_ON(cl == cldev->cl);
++ scoped_guard(mutex, &bus->device_lock) {
++ mei_cl_flush_queues(cldev->cl, NULL);
++ mei_me_cl_put(cldev->me_cl);
++ list_for_each_entry(cl, &bus->file_list, link)
++ WARN_ON(cl == cldev->cl);
++ }
++ mei_dev_bus_put(bus);
+
+ kfree(cldev->cl);
+ kfree(cldev);
+diff --git a/drivers/misc/nsm.c b/drivers/misc/nsm.c
+index ef7b3274234099..3960506eb7ab3d 100644
+--- a/drivers/misc/nsm.c
++++ b/drivers/misc/nsm.c
+@@ -367,7 +367,7 @@ static long nsm_dev_ioctl(struct file *file, unsigned int cmd,
+ /* Copy user argument struct to kernel argument struct */
+ r = -EFAULT;
+ if (copy_from_user(&raw, argp, _IOC_SIZE(cmd)))
+- goto out;
++ return r;
+
+ mutex_lock(&nsm->lock);
+
+@@ -413,6 +413,7 @@ static int nsm_device_init_vq(struct virtio_device *vdev)
+ }
+
+ static const struct file_operations nsm_dev_fops = {
++ .owner = THIS_MODULE,
+ .unlocked_ioctl = nsm_dev_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ };
+diff --git a/drivers/mtd/maps/vmu-flash.c b/drivers/mtd/maps/vmu-flash.c
+index 3667faefe6c58f..e992f4d1417e6f 100644
+--- a/drivers/mtd/maps/vmu-flash.c
++++ b/drivers/mtd/maps/vmu-flash.c
+@@ -610,7 +610,7 @@ static int vmu_connect(struct maple_device *mdev)
+
+ basic_flash_data = be32_to_cpu(mdev->devinfo.function_data[c - 1]);
+
+- card = kmalloc(sizeof(struct memcard), GFP_KERNEL);
++ card = kzalloc_obj(struct memcard);
+ if (!card) {
+ error = -ENOMEM;
+ goto fail_nomem;
+@@ -628,15 +628,13 @@ static int vmu_connect(struct maple_device *mdev)
+ * Not sure there are actually any multi-partition devices in the
+ * real world, but the hardware supports them, so, so will we
+ */
+- card->parts = kmalloc_array(card->partitions, sizeof(struct vmupart),
+- GFP_KERNEL);
++ card->parts = kzalloc_objs(struct vmupart, card->partitions);
+ if (!card->parts) {
+ error = -ENOMEM;
+ goto fail_partitions;
+ }
+
+- card->mtd = kmalloc_array(card->partitions, sizeof(struct mtd_info),
+- GFP_KERNEL);
++ card->mtd = kzalloc_objs(struct mtd_info, card->partitions);
+ if (!card->mtd) {
+ error = -ENOMEM;
+ goto fail_mtd_info;
+diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
+index 724f917f91bac0..0c8e8eb5f0f3d8 100644
+--- a/drivers/mtd/mtdcore.c
++++ b/drivers/mtd/mtdcore.c
+@@ -104,6 +104,15 @@ static void mtd_release(struct device *dev)
+ device_destroy(&mtd_class, index + 1);
+ }
+
++/*
++ * No-op device release used in add_mtd_device() error paths.
++ * Prevents mtd_release() from being called via device_release(),
++ * which would free the mtd_info that the caller still manages.
++ */
++static void mtd_dev_release_nop(struct device *dev)
++{
++}
++
+ static void mtd_device_release(struct kref *kref)
+ {
+ struct mtd_info *mtd = container_of(kref, struct mtd_info, refcnt);
+@@ -746,10 +755,8 @@ int add_mtd_device(struct mtd_info *mtd)
+ mtd_check_of_node(mtd);
+ of_node_get(mtd_get_of_node(mtd));
+ error = device_register(&mtd->dev);
+- if (error) {
+- put_device(&mtd->dev);
++ if (error)
+ goto fail_added;
+- }
+
+ /* Add the nvmem provider */
+ error = mtd_nvmem_add(mtd);
+@@ -787,8 +794,16 @@ int add_mtd_device(struct mtd_info *mtd)
+ return 0;
+
+ fail_nvmem_add:
+- device_unregister(&mtd->dev);
++ device_del(&mtd->dev);
+ fail_added:
++ /*
++ * Clear type and set nop release to prevent mtd_release() ->
++ * release_mtd_partition() -> free_partition() from freeing mtd.
++ * The caller handles cleanup on failure.
++ */
++ mtd->dev.type = NULL;
++ mtd->dev.release = mtd_dev_release_nop;
++ put_device(&mtd->dev);
+ of_node_put(mtd_get_of_node(mtd));
+ idr_remove(&mtd_idr, i);
+ fail_locked:
+diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c
+index 680366616da240..4d695875ea1b23 100644
+--- a/drivers/mtd/mtdswap.c
++++ b/drivers/mtd/mtdswap.c
+@@ -125,6 +125,7 @@ struct mtdswap_dev {
+
+ char *page_buf;
+ char *oob_buf;
++ struct dentry *debugfs_stats;
+ };
+
+ struct mtdswap_oobdata {
+@@ -1262,7 +1263,8 @@ static int mtdswap_add_debugfs(struct mtdswap_dev *d)
+ if (IS_ERR_OR_NULL(root))
+ return -1;
+
+- debugfs_create_file("mtdswap_stats", S_IRUSR, root, d, &mtdswap_fops);
++ d->debugfs_stats = debugfs_create_file("mtdswap_stats", 0400, root,
++ d, &mtdswap_fops);
+
+ return 0;
+ }
+@@ -1463,6 +1465,7 @@ static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev)
+ {
+ struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
+
++ debugfs_remove(d->debugfs_stats);
+ del_mtd_blktrans_dev(dev);
+ mtdswap_cleanup(d);
+ kfree(d);
+diff --git a/drivers/mtd/nand/ecc-mtk.c b/drivers/mtd/nand/ecc-mtk.c
+index c75bb8b80cc1e1..96703f0a418ea2 100644
+--- a/drivers/mtd/nand/ecc-mtk.c
++++ b/drivers/mtd/nand/ecc-mtk.c
+@@ -123,8 +123,8 @@ static int mt7622_ecc_regs[] = {
+ [ECC_DECIRQ_STA] = 0x144,
+ };
+
+-static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc,
+- enum mtk_ecc_operation op)
++static inline int mtk_ecc_wait_idle(struct mtk_ecc *ecc,
++ enum mtk_ecc_operation op)
+ {
+ struct device *dev = ecc->dev;
+ u32 val;
+@@ -136,6 +136,8 @@ static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc,
+ if (ret)
+ dev_warn(dev, "%s NOT idle\n",
+ op == ECC_ENCODE ? "encoder" : "decoder");
++
++ return ret;
+ }
+
+ static irqreturn_t mtk_ecc_irq(int irq, void *id)
+@@ -312,7 +314,11 @@ int mtk_ecc_enable(struct mtk_ecc *ecc, struct mtk_ecc_config *config)
+ return ret;
+ }
+
+- mtk_ecc_wait_idle(ecc, op);
++ ret = mtk_ecc_wait_idle(ecc, op);
++ if (ret) {
++ mutex_unlock(&ecc->lock);
++ return ret;
++ }
+
+ ret = mtk_ecc_config(ecc, config);
+ if (ret) {
+@@ -412,7 +418,9 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config,
+ if (ret)
+ goto timeout;
+
+- mtk_ecc_wait_idle(ecc, ECC_ENCODE);
++ ret = mtk_ecc_wait_idle(ecc, ECC_ENCODE);
++ if (ret)
++ goto timeout;
+
+ /* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */
+ len = (config->strength * ecc->caps->parity_bits + 7) >> 3;
+diff --git a/drivers/net/amt.c b/drivers/net/amt.c
+index ef6848c2e60841..b2572e443e43ac 100644
+--- a/drivers/net/amt.c
++++ b/drivers/net/amt.c
+@@ -1206,7 +1206,7 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev)
+ data = true;
+ }
+ v6 = false;
+- group.ip4 = iph->daddr;
++ group.ip4 = ip_hdr(skb)->daddr;
+ #if IS_ENABLED(CONFIG_IPV6)
+ } else if (iph->version == 6) {
+ ip6h = ipv6_hdr(skb);
+@@ -1230,7 +1230,7 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev)
+ data = true;
+ }
+ v6 = true;
+- group.ip6 = ip6h->daddr;
++ group.ip6 = ipv6_hdr(skb)->daddr;
+ #endif
+ } else {
+ dev->stats.tx_errors++;
+@@ -1273,12 +1273,12 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev)
+ hlist_for_each_entry_rcu(gnode, &tunnel->groups[hash],
+ node) {
+ if (!v6) {
+- if (gnode->group_addr.ip4 == iph->daddr)
++ if (gnode->group_addr.ip4 == group.ip4)
+ goto found;
+ #if IS_ENABLED(CONFIG_IPV6)
+ } else {
+ if (ipv6_addr_equal(&gnode->group_addr.ip6,
+- &ip6h->daddr))
++ &group.ip6))
+ goto found;
+ #endif
+ }
+@@ -1995,14 +1995,18 @@ static void amt_igmpv3_report_handler(struct amt_dev *amt, struct sk_buff *skb,
+ struct igmpv3_report *ihrv3 = igmpv3_report_hdr(skb);
+ int len = skb_transport_offset(skb) + sizeof(*ihrv3);
+ void *zero_grec = (void *)&igmpv3_zero_grec;
+- struct iphdr *iph = ip_hdr(skb);
+ struct amt_group_node *gnode;
+ union amt_addr group, host;
+ struct igmpv3_grec *grec;
++ __be32 saddr;
+ u16 nsrcs;
++ u16 ngrec;
+ int i;
+
+- for (i = 0; i < ntohs(ihrv3->ngrec); i++) {
++ saddr = ip_hdr(skb)->saddr;
++ ngrec = ntohs(ihrv3->ngrec);
++
++ for (i = 0; i < ngrec; i++) {
+ len += sizeof(*grec);
+ if (!ip_mc_may_pull(skb, len))
+ break;
+@@ -2014,10 +2018,13 @@ static void amt_igmpv3_report_handler(struct amt_dev *amt, struct sk_buff *skb,
+ if (!ip_mc_may_pull(skb, len))
+ break;
+
++ grec = (void *)(skb->data + len - sizeof(*grec) -
++ nsrcs * sizeof(__be32));
++
+ memset(&group, 0, sizeof(union amt_addr));
+ group.ip4 = grec->grec_mca;
+ memset(&host, 0, sizeof(union amt_addr));
+- host.ip4 = iph->saddr;
++ host.ip4 = saddr;
+ gnode = amt_lookup_group(tunnel, &group, &host, false);
+ if (!gnode) {
+ gnode = amt_add_group(amt, tunnel, &group, &host,
+@@ -2157,14 +2164,18 @@ static void amt_mldv2_report_handler(struct amt_dev *amt, struct sk_buff *skb,
+ struct mld2_report *mld2r = (struct mld2_report *)icmp6_hdr(skb);
+ int len = skb_transport_offset(skb) + sizeof(*mld2r);
+ void *zero_grec = (void *)&mldv2_zero_grec;
+- struct ipv6hdr *ip6h = ipv6_hdr(skb);
+ struct amt_group_node *gnode;
+ union amt_addr group, host;
+ struct mld2_grec *grec;
++ struct in6_addr saddr;
+ u16 nsrcs;
++ u16 ngrec;
+ int i;
+
+- for (i = 0; i < ntohs(mld2r->mld2r_ngrec); i++) {
++ saddr = ipv6_hdr(skb)->saddr;
++ ngrec = ntohs(mld2r->mld2r_ngrec);
++
++ for (i = 0; i < ngrec; i++) {
+ len += sizeof(*grec);
+ if (!ipv6_mc_may_pull(skb, len))
+ break;
+@@ -2176,10 +2187,13 @@ static void amt_mldv2_report_handler(struct amt_dev *amt, struct sk_buff *skb,
+ if (!ipv6_mc_may_pull(skb, len))
+ break;
+
++ grec = (void *)(skb->data + len - sizeof(*grec) -
++ nsrcs * sizeof(struct in6_addr));
++
+ memset(&group, 0, sizeof(union amt_addr));
+ group.ip6 = grec->grec_mca;
+ memset(&host, 0, sizeof(union amt_addr));
+- host.ip6 = ip6h->saddr;
++ host.ip6 = saddr;
+ gnode = amt_lookup_group(tunnel, &group, &host, true);
+ if (!gnode) {
+ gnode = amt_add_group(amt, tunnel, &group, &host,
+@@ -2300,7 +2314,9 @@ static bool amt_multicast_data_handler(struct amt_dev *amt, struct sk_buff *skb)
+ skb_push(skb, sizeof(*eth));
+ skb_reset_mac_header(skb);
+ skb_pull(skb, sizeof(*eth));
+- eth = eth_hdr(skb);
++
++ if (skb_cow_head(skb, 0))
++ return true;
+
+ if (!pskb_may_pull(skb, sizeof(*iph)))
+ return true;
+@@ -2310,6 +2326,7 @@ static bool amt_multicast_data_handler(struct amt_dev *amt, struct sk_buff *skb)
+ if (!ipv4_is_multicast(iph->daddr))
+ return true;
+ skb->protocol = htons(ETH_P_IP);
++ eth = eth_hdr(skb);
+ eth->h_proto = htons(ETH_P_IP);
+ ip_eth_mc_map(iph->daddr, eth->h_dest);
+ #if IS_ENABLED(CONFIG_IPV6)
+@@ -2323,6 +2340,7 @@ static bool amt_multicast_data_handler(struct amt_dev *amt, struct sk_buff *skb)
+ if (!ipv6_addr_is_multicast(&ip6h->daddr))
+ return true;
+ skb->protocol = htons(ETH_P_IPV6);
++ eth = eth_hdr(skb);
+ eth->h_proto = htons(ETH_P_IPV6);
+ ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest);
+ #endif
+@@ -2346,10 +2364,12 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
+ struct sk_buff *skb)
+ {
+ struct amt_header_membership_query *amtmq;
+- struct igmpv3_query *ihv3;
+ struct ethhdr *eth, *oeth;
++ struct igmpv3_query *ihv3;
++ u8 h_source[ETH_ALEN];
+ struct iphdr *iph;
+ int hdr_size, len;
++ u64 response_mac;
+
+ hdr_size = sizeof(*amtmq) + sizeof(struct udphdr);
+ if (!pskb_may_pull(skb, hdr_size))
+@@ -2362,6 +2382,8 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
+ if (amtmq->nonce != amt->nonce)
+ return true;
+
++ response_mac = amtmq->response_mac;
++
+ hdr_size -= sizeof(*eth);
+ if (iptunnel_pull_header(skb, hdr_size, htons(ETH_P_TEB), false))
+ return true;
+@@ -2371,6 +2393,9 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
+ skb_pull(skb, sizeof(*eth));
+ skb_reset_network_header(skb);
+ eth = eth_hdr(skb);
++ ether_addr_copy(h_source, oeth->h_source);
++ if (skb_cow_head(skb, 0))
++ return true;
+ if (!pskb_may_pull(skb, sizeof(*iph)))
+ return true;
+
+@@ -2383,6 +2408,7 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
+ sizeof(*ihv3)))
+ return true;
+
++ iph = ip_hdr(skb);
+ if (!ipv4_is_multicast(iph->daddr))
+ return true;
+
+@@ -2390,10 +2416,11 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
+ skb_reset_transport_header(skb);
+ skb_push(skb, sizeof(*iph) + AMT_IPHDR_OPTS);
+ WRITE_ONCE(amt->ready4, true);
+- amt->mac = amtmq->response_mac;
++ amt->mac = response_mac;
+ amt->req_cnt = 0;
+ amt->qi = ihv3->qqic;
+ skb->protocol = htons(ETH_P_IP);
++ eth = eth_hdr(skb);
+ eth->h_proto = htons(ETH_P_IP);
+ ip_eth_mc_map(iph->daddr, eth->h_dest);
+ #if IS_ENABLED(CONFIG_IPV6)
+@@ -2416,10 +2443,11 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
+ skb_reset_transport_header(skb);
+ skb_push(skb, sizeof(*ip6h) + AMT_IP6HDR_OPTS);
+ WRITE_ONCE(amt->ready6, true);
+- amt->mac = amtmq->response_mac;
++ amt->mac = response_mac;
+ amt->req_cnt = 0;
+ amt->qi = mld2q->mld2q_qqic;
+ skb->protocol = htons(ETH_P_IPV6);
++ eth = eth_hdr(skb);
+ eth->h_proto = htons(ETH_P_IPV6);
+ ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest);
+ #endif
+@@ -2427,7 +2455,7 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
+ return true;
+ }
+
+- ether_addr_copy(eth->h_source, oeth->h_source);
++ ether_addr_copy(eth->h_source, h_source);
+ skb->pkt_type = PACKET_MULTICAST;
+ skb->ip_summed = CHECKSUM_NONE;
+ len = skb->len;
+@@ -2450,8 +2478,11 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb)
+ struct ethhdr *eth;
+ struct iphdr *iph;
+ int len, hdr_size;
++ u64 response_mac;
++ __be32 saddr;
++ __be32 nonce;
+
+- iph = ip_hdr(skb);
++ saddr = ip_hdr(skb)->saddr;
+
+ hdr_size = sizeof(*amtmu) + sizeof(struct udphdr);
+ if (!pskb_may_pull(skb, hdr_size))
+@@ -2461,15 +2492,18 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb)
+ if (amtmu->reserved || amtmu->version)
+ return true;
+
++ nonce = amtmu->nonce;
++ response_mac = amtmu->response_mac;
++
+ if (iptunnel_pull_header(skb, hdr_size, skb->protocol, false))
+ return true;
+
+ skb_reset_network_header(skb);
+
+ list_for_each_entry_rcu(tunnel, &amt->tunnel_list, list) {
+- if (tunnel->ip4 == iph->saddr) {
+- if ((amtmu->nonce == tunnel->nonce &&
+- amtmu->response_mac == tunnel->mac)) {
++ if (tunnel->ip4 == saddr) {
++ if ((nonce == tunnel->nonce &&
++ response_mac == tunnel->mac)) {
+ mod_delayed_work(amt_wq, &tunnel->gc_wq,
+ msecs_to_jiffies(amt_gmi(amt))
+ * 3);
+@@ -2487,6 +2521,9 @@ report:
+ if (!pskb_may_pull(skb, sizeof(*iph)))
+ return true;
+
++ if (skb_cow_head(skb, 0))
++ return true;
++
+ iph = ip_hdr(skb);
+ if (iph->version == 4) {
+ if (ip_mc_check_igmp(skb)) {
+@@ -2503,6 +2540,7 @@ report:
+ eth = eth_hdr(skb);
+ skb->protocol = htons(ETH_P_IP);
+ eth->h_proto = htons(ETH_P_IP);
++ iph = ip_hdr(skb);
+ ip_eth_mc_map(iph->daddr, eth->h_dest);
+ #if IS_ENABLED(CONFIG_IPV6)
+ } else if (iph->version == 6) {
+@@ -2522,6 +2560,7 @@ report:
+ eth = eth_hdr(skb);
+ skb->protocol = htons(ETH_P_IPV6);
+ eth->h_proto = htons(ETH_P_IPV6);
++ ip6h = ipv6_hdr(skb);
+ ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest);
+ #endif
+ } else {
+@@ -2767,7 +2806,7 @@ drop:
+ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
+ {
+ struct amt_dev *amt;
+- struct iphdr *iph;
++ __be32 saddr;
+ int type;
+ bool err;
+
+@@ -2780,7 +2819,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
+ }
+
+ skb->dev = amt->dev;
+- iph = ip_hdr(skb);
++ saddr = ip_hdr(skb)->saddr;
+ type = amt_parse_type(skb);
+ if (type == -1) {
+ err = true;
+@@ -2790,7 +2829,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
+ if (amt->mode == AMT_MODE_GATEWAY) {
+ switch (type) {
+ case AMT_MSG_ADVERTISEMENT:
+- if (iph->saddr != amt->discovery_ip) {
++ if (saddr != amt->discovery_ip) {
+ netdev_dbg(amt->dev, "Invalid Relay IP\n");
+ err = true;
+ goto drop;
+@@ -2802,7 +2841,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
+ }
+ goto out;
+ case AMT_MSG_MULTICAST_DATA:
+- if (iph->saddr != amt->remote_ip) {
++ if (saddr != amt->remote_ip) {
+ netdev_dbg(amt->dev, "Invalid Relay IP\n");
+ err = true;
+ goto drop;
+@@ -2813,7 +2852,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
+ else
+ goto out;
+ case AMT_MSG_MEMBERSHIP_QUERY:
+- if (iph->saddr != amt->remote_ip) {
++ if (saddr != amt->remote_ip) {
+ netdev_dbg(amt->dev, "Invalid Relay IP\n");
+ err = true;
+ goto drop;
+@@ -2990,9 +3029,15 @@ static int amt_dev_open(struct net_device *dev)
+ amt->event_idx = 0;
+ amt->nr_events = 0;
+
++ enable_delayed_work(&amt->discovery_wq);
++ enable_delayed_work(&amt->req_wq);
++
+ err = amt_socket_create(amt);
+- if (err)
++ if (err) {
++ disable_delayed_work(&amt->req_wq);
++ disable_delayed_work(&amt->discovery_wq);
+ return err;
++ }
+
+ amt->req_cnt = 0;
+ amt->remote_ip = 0;
+@@ -3018,8 +3063,8 @@ static int amt_dev_stop(struct net_device *dev)
+ struct sk_buff *skb;
+ int i;
+
+- cancel_delayed_work_sync(&amt->req_wq);
+- cancel_delayed_work_sync(&amt->discovery_wq);
++ disable_delayed_work_sync(&amt->req_wq);
++ disable_delayed_work_sync(&amt->discovery_wq);
+ cancel_delayed_work_sync(&amt->secret_wq);
+
+ /* shutdown */
+@@ -3274,6 +3319,8 @@ static int amt_newlink(struct net *net, struct net_device *dev,
+ INIT_DELAYED_WORK(&amt->req_wq, amt_req_work);
+ INIT_DELAYED_WORK(&amt->secret_wq, amt_secret_work);
+ INIT_WORK(&amt->event_wq, amt_event_work);
++ disable_delayed_work(&amt->req_wq);
++ disable_delayed_work(&amt->discovery_wq);
+ INIT_LIST_HEAD(&amt->tunnel_list);
+ return 0;
+ err:
+diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
+index 7d71cfc2c93741..2919329a8c2006 100644
+--- a/drivers/net/bonding/bond_main.c
++++ b/drivers/net/bonding/bond_main.c
+@@ -3512,7 +3512,8 @@ static void bond_send_validate(struct bonding *bond, struct slave *slave)
+ {
+ bond_arp_send_all(bond, slave);
+ #if IS_ENABLED(CONFIG_IPV6)
+- bond_ns_send_all(bond, slave);
++ if (likely(ipv6_mod_enabled()))
++ bond_ns_send_all(bond, slave);
+ #endif
+ }
+
+diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
+index 9a542e3c9b05d8..af09a658cbf6af 100644
+--- a/drivers/net/ethernet/Kconfig
++++ b/drivers/net/ethernet/Kconfig
+@@ -20,6 +20,7 @@ source "drivers/net/ethernet/actions/Kconfig"
+ source "drivers/net/ethernet/adaptec/Kconfig"
+ source "drivers/net/ethernet/aeroflex/Kconfig"
+ source "drivers/net/ethernet/agere/Kconfig"
++source "drivers/net/ethernet/airoha/Kconfig"
+ source "drivers/net/ethernet/alacritech/Kconfig"
+ source "drivers/net/ethernet/allwinner/Kconfig"
+ source "drivers/net/ethernet/alteon/Kconfig"
+diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
+index 99fa180dedb805..67182339469a0d 100644
+--- a/drivers/net/ethernet/Makefile
++++ b/drivers/net/ethernet/Makefile
+@@ -10,6 +10,7 @@ obj-$(CONFIG_NET_VENDOR_ADAPTEC) += adaptec/
+ obj-$(CONFIG_GRETH) += aeroflex/
+ obj-$(CONFIG_NET_VENDOR_ADI) += adi/
+ obj-$(CONFIG_NET_VENDOR_AGERE) += agere/
++obj-$(CONFIG_NET_VENDOR_AIROHA) += airoha/
+ obj-$(CONFIG_NET_VENDOR_ALACRITECH) += alacritech/
+ obj-$(CONFIG_NET_VENDOR_ALLWINNER) += allwinner/
+ obj-$(CONFIG_NET_VENDOR_ALTEON) += alteon/
+diff --git a/drivers/net/ethernet/airoha/Kconfig b/drivers/net/ethernet/airoha/Kconfig
+new file mode 100644
+index 00000000000000..b6a131845f13b2
+--- /dev/null
++++ b/drivers/net/ethernet/airoha/Kconfig
+@@ -0,0 +1,18 @@
++# SPDX-License-Identifier: GPL-2.0-only
++config NET_VENDOR_AIROHA
++ bool "Airoha devices"
++ depends on ARCH_AIROHA || COMPILE_TEST
++ help
++ If you have a Airoha SoC with ethernet, say Y.
++
++if NET_VENDOR_AIROHA
++
++config NET_AIROHA
++ tristate "Airoha SoC Gigabit Ethernet support"
++ depends on NET_DSA || !NET_DSA
++ select PAGE_POOL
++ help
++ This driver supports the gigabit ethernet MACs in the
++ Airoha SoC family.
++
++endif #NET_VENDOR_AIROHA
+diff --git a/drivers/net/ethernet/airoha/Makefile b/drivers/net/ethernet/airoha/Makefile
+new file mode 100644
+index 00000000000000..73a6f3680a4c4c
+--- /dev/null
++++ b/drivers/net/ethernet/airoha/Makefile
+@@ -0,0 +1,6 @@
++# SPDX-License-Identifier: GPL-2.0-only
++#
++# Airoha for the Mediatek SoCs built-in ethernet macs
++#
++
++obj-$(CONFIG_NET_AIROHA) += airoha_eth.o
+diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
+new file mode 100644
+index 00000000000000..58e5791e9cd4d4
+--- /dev/null
++++ b/drivers/net/ethernet/airoha/airoha_eth.c
+@@ -0,0 +1,3059 @@
++// SPDX-License-Identifier: GPL-2.0-only
++/*
++ * Copyright (c) 2024 AIROHA Inc
++ * Author: Lorenzo Bianconi <[email protected]>
++ */
++#include <linux/etherdevice.h>
++#include <linux/iopoll.h>
++#include <linux/kernel.h>
++#include <linux/netdevice.h>
++#include <linux/of.h>
++#include <linux/of_net.h>
++#include <linux/platform_device.h>
++#include <linux/reset.h>
++#include <linux/tcp.h>
++#include <linux/u64_stats_sync.h>
++#include <net/dsa.h>
++#include <net/page_pool/helpers.h>
++#include <net/pkt_cls.h>
++#include <uapi/linux/ppp_defs.h>
++
++#define AIROHA_MAX_NUM_GDM_PORTS 1
++#define AIROHA_MAX_NUM_QDMA 2
++#define AIROHA_MAX_NUM_RSTS 3
++#define AIROHA_MAX_NUM_XSI_RSTS 5
++#define AIROHA_MAX_MTU 2000
++#define AIROHA_MAX_PACKET_SIZE 2048
++#define AIROHA_NUM_QOS_CHANNELS 4
++#define AIROHA_NUM_QOS_QUEUES 8
++#define AIROHA_NUM_TX_RING 32
++#define AIROHA_NUM_RX_RING 32
++#define AIROHA_FE_MC_MAX_VLAN_TABLE 64
++#define AIROHA_FE_MC_MAX_VLAN_PORT 16
++#define AIROHA_NUM_TX_IRQ 2
++#define HW_DSCP_NUM 2048
++#define IRQ_QUEUE_LEN(_n) ((_n) ? 1024 : 2048)
++#define TX_DSCP_NUM 1024
++#define RX_DSCP_NUM(_n) \
++ ((_n) == 2 ? 128 : \
++ (_n) == 11 ? 128 : \
++ (_n) == 15 ? 128 : \
++ (_n) == 0 ? 1024 : 16)
++
++#define PSE_RSV_PAGES 128
++#define PSE_QUEUE_RSV_PAGES 64
++
++/* FE */
++#define PSE_BASE 0x0100
++#define CSR_IFC_BASE 0x0200
++#define CDM1_BASE 0x0400
++#define GDM1_BASE 0x0500
++#define PPE1_BASE 0x0c00
++
++#define CDM2_BASE 0x1400
++#define GDM2_BASE 0x1500
++
++#define GDM3_BASE 0x1100
++#define GDM4_BASE 0x2500
++
++#define GDM_BASE(_n) \
++ ((_n) == 4 ? GDM4_BASE : \
++ (_n) == 3 ? GDM3_BASE : \
++ (_n) == 2 ? GDM2_BASE : GDM1_BASE)
++
++#define REG_FE_DMA_GLO_CFG 0x0000
++#define FE_DMA_GLO_L2_SPACE_MASK GENMASK(7, 4)
++#define FE_DMA_GLO_PG_SZ_MASK BIT(3)
++
++#define REG_FE_RST_GLO_CFG 0x0004
++#define FE_RST_GDM4_MBI_ARB_MASK BIT(3)
++#define FE_RST_GDM3_MBI_ARB_MASK BIT(2)
++#define FE_RST_CORE_MASK BIT(0)
++
++#define REG_FE_WAN_MAC_H 0x0030
++#define REG_FE_LAN_MAC_H 0x0040
++
++#define REG_FE_MAC_LMIN(_n) ((_n) + 0x04)
++#define REG_FE_MAC_LMAX(_n) ((_n) + 0x08)
++
++#define REG_FE_CDM1_OQ_MAP0 0x0050
++#define REG_FE_CDM1_OQ_MAP1 0x0054
++#define REG_FE_CDM1_OQ_MAP2 0x0058
++#define REG_FE_CDM1_OQ_MAP3 0x005c
++
++#define REG_FE_PCE_CFG 0x0070
++#define PCE_DPI_EN_MASK BIT(2)
++#define PCE_KA_EN_MASK BIT(1)
++#define PCE_MC_EN_MASK BIT(0)
++
++#define REG_FE_PSE_QUEUE_CFG_WR 0x0080
++#define PSE_CFG_PORT_ID_MASK GENMASK(27, 24)
++#define PSE_CFG_QUEUE_ID_MASK GENMASK(20, 16)
++#define PSE_CFG_WR_EN_MASK BIT(8)
++#define PSE_CFG_OQRSV_SEL_MASK BIT(0)
++
++#define REG_FE_PSE_QUEUE_CFG_VAL 0x0084
++#define PSE_CFG_OQ_RSV_MASK GENMASK(13, 0)
++
++#define PSE_FQ_CFG 0x008c
++#define PSE_FQ_LIMIT_MASK GENMASK(14, 0)
++
++#define REG_FE_PSE_BUF_SET 0x0090
++#define PSE_SHARE_USED_LTHD_MASK GENMASK(31, 16)
++#define PSE_ALLRSV_MASK GENMASK(14, 0)
++
++#define REG_PSE_SHARE_USED_THD 0x0094
++#define PSE_SHARE_USED_MTHD_MASK GENMASK(31, 16)
++#define PSE_SHARE_USED_HTHD_MASK GENMASK(15, 0)
++
++#define REG_GDM_MISC_CFG 0x0148
++#define GDM2_RDM_ACK_WAIT_PREF_MASK BIT(9)
++#define GDM2_CHN_VLD_MODE_MASK BIT(5)
++
++#define REG_FE_CSR_IFC_CFG CSR_IFC_BASE
++#define FE_IFC_EN_MASK BIT(0)
++
++#define REG_FE_VIP_PORT_EN 0x01f0
++#define REG_FE_IFC_PORT_EN 0x01f4
++
++#define REG_PSE_IQ_REV1 (PSE_BASE + 0x08)
++#define PSE_IQ_RES1_P2_MASK GENMASK(23, 16)
++
++#define REG_PSE_IQ_REV2 (PSE_BASE + 0x0c)
++#define PSE_IQ_RES2_P5_MASK GENMASK(15, 8)
++#define PSE_IQ_RES2_P4_MASK GENMASK(7, 0)
++
++#define REG_FE_VIP_EN(_n) (0x0300 + ((_n) << 3))
++#define PATN_FCPU_EN_MASK BIT(7)
++#define PATN_SWP_EN_MASK BIT(6)
++#define PATN_DP_EN_MASK BIT(5)
++#define PATN_SP_EN_MASK BIT(4)
++#define PATN_TYPE_MASK GENMASK(3, 1)
++#define PATN_EN_MASK BIT(0)
++
++#define REG_FE_VIP_PATN(_n) (0x0304 + ((_n) << 3))
++#define PATN_DP_MASK GENMASK(31, 16)
++#define PATN_SP_MASK GENMASK(15, 0)
++
++#define REG_CDM1_VLAN_CTRL CDM1_BASE
++#define CDM1_VLAN_MASK GENMASK(31, 16)
++
++#define REG_CDM1_FWD_CFG (CDM1_BASE + 0x08)
++#define CDM1_VIP_QSEL_MASK GENMASK(24, 20)
++
++#define REG_CDM1_CRSN_QSEL(_n) (CDM1_BASE + 0x10 + ((_n) << 2))
++#define CDM1_CRSN_QSEL_REASON_MASK(_n) \
++ GENMASK(4 + (((_n) % 4) << 3), (((_n) % 4) << 3))
++
++#define REG_CDM2_FWD_CFG (CDM2_BASE + 0x08)
++#define CDM2_OAM_QSEL_MASK GENMASK(31, 27)
++#define CDM2_VIP_QSEL_MASK GENMASK(24, 20)
++
++#define REG_CDM2_CRSN_QSEL(_n) (CDM2_BASE + 0x10 + ((_n) << 2))
++#define CDM2_CRSN_QSEL_REASON_MASK(_n) \
++ GENMASK(4 + (((_n) % 4) << 3), (((_n) % 4) << 3))
++
++#define REG_GDM_FWD_CFG(_n) GDM_BASE(_n)
++#define GDM_DROP_CRC_ERR BIT(23)
++#define GDM_IP4_CKSUM BIT(22)
++#define GDM_TCP_CKSUM BIT(21)
++#define GDM_UDP_CKSUM BIT(20)
++#define GDM_UCFQ_MASK GENMASK(15, 12)
++#define GDM_BCFQ_MASK GENMASK(11, 8)
++#define GDM_MCFQ_MASK GENMASK(7, 4)
++#define GDM_OCFQ_MASK GENMASK(3, 0)
++
++#define REG_GDM_INGRESS_CFG(_n) (GDM_BASE(_n) + 0x10)
++#define GDM_INGRESS_FC_EN_MASK BIT(1)
++#define GDM_STAG_EN_MASK BIT(0)
++
++#define REG_GDM_LEN_CFG(_n) (GDM_BASE(_n) + 0x14)
++#define GDM_SHORT_LEN_MASK GENMASK(13, 0)
++#define GDM_LONG_LEN_MASK GENMASK(29, 16)
++
++#define REG_FE_CPORT_CFG (GDM1_BASE + 0x40)
++#define FE_CPORT_PAD BIT(26)
++#define FE_CPORT_PORT_XFC_MASK BIT(25)
++#define FE_CPORT_QUEUE_XFC_MASK BIT(24)
++
++#define REG_FE_GDM_MIB_CLEAR(_n) (GDM_BASE(_n) + 0xf0)
++#define FE_GDM_MIB_RX_CLEAR_MASK BIT(1)
++#define FE_GDM_MIB_TX_CLEAR_MASK BIT(0)
++
++#define REG_FE_GDM1_MIB_CFG (GDM1_BASE + 0xf4)
++#define FE_STRICT_RFC2819_MODE_MASK BIT(31)
++#define FE_GDM1_TX_MIB_SPLIT_EN_MASK BIT(17)
++#define FE_GDM1_RX_MIB_SPLIT_EN_MASK BIT(16)
++#define FE_TX_MIB_ID_MASK GENMASK(15, 8)
++#define FE_RX_MIB_ID_MASK GENMASK(7, 0)
++
++#define REG_FE_GDM_TX_OK_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x104)
++#define REG_FE_GDM_TX_OK_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x10c)
++#define REG_FE_GDM_TX_ETH_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x110)
++#define REG_FE_GDM_TX_ETH_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x114)
++#define REG_FE_GDM_TX_ETH_DROP_CNT(_n) (GDM_BASE(_n) + 0x118)
++#define REG_FE_GDM_TX_ETH_BC_CNT(_n) (GDM_BASE(_n) + 0x11c)
++#define REG_FE_GDM_TX_ETH_MC_CNT(_n) (GDM_BASE(_n) + 0x120)
++#define REG_FE_GDM_TX_ETH_RUNT_CNT(_n) (GDM_BASE(_n) + 0x124)
++#define REG_FE_GDM_TX_ETH_LONG_CNT(_n) (GDM_BASE(_n) + 0x128)
++#define REG_FE_GDM_TX_ETH_E64_CNT_L(_n) (GDM_BASE(_n) + 0x12c)
++#define REG_FE_GDM_TX_ETH_L64_CNT_L(_n) (GDM_BASE(_n) + 0x130)
++#define REG_FE_GDM_TX_ETH_L127_CNT_L(_n) (GDM_BASE(_n) + 0x134)
++#define REG_FE_GDM_TX_ETH_L255_CNT_L(_n) (GDM_BASE(_n) + 0x138)
++#define REG_FE_GDM_TX_ETH_L511_CNT_L(_n) (GDM_BASE(_n) + 0x13c)
++#define REG_FE_GDM_TX_ETH_L1023_CNT_L(_n) (GDM_BASE(_n) + 0x140)
++
++#define REG_FE_GDM_RX_OK_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x148)
++#define REG_FE_GDM_RX_FC_DROP_CNT(_n) (GDM_BASE(_n) + 0x14c)
++#define REG_FE_GDM_RX_RC_DROP_CNT(_n) (GDM_BASE(_n) + 0x150)
++#define REG_FE_GDM_RX_OVERFLOW_DROP_CNT(_n) (GDM_BASE(_n) + 0x154)
++#define REG_FE_GDM_RX_ERROR_DROP_CNT(_n) (GDM_BASE(_n) + 0x158)
++#define REG_FE_GDM_RX_OK_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x15c)
++#define REG_FE_GDM_RX_ETH_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x160)
++#define REG_FE_GDM_RX_ETH_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x164)
++#define REG_FE_GDM_RX_ETH_DROP_CNT(_n) (GDM_BASE(_n) + 0x168)
++#define REG_FE_GDM_RX_ETH_BC_CNT(_n) (GDM_BASE(_n) + 0x16c)
++#define REG_FE_GDM_RX_ETH_MC_CNT(_n) (GDM_BASE(_n) + 0x170)
++#define REG_FE_GDM_RX_ETH_CRC_ERR_CNT(_n) (GDM_BASE(_n) + 0x174)
++#define REG_FE_GDM_RX_ETH_FRAG_CNT(_n) (GDM_BASE(_n) + 0x178)
++#define REG_FE_GDM_RX_ETH_JABBER_CNT(_n) (GDM_BASE(_n) + 0x17c)
++#define REG_FE_GDM_RX_ETH_RUNT_CNT(_n) (GDM_BASE(_n) + 0x180)
++#define REG_FE_GDM_RX_ETH_LONG_CNT(_n) (GDM_BASE(_n) + 0x184)
++#define REG_FE_GDM_RX_ETH_E64_CNT_L(_n) (GDM_BASE(_n) + 0x188)
++#define REG_FE_GDM_RX_ETH_L64_CNT_L(_n) (GDM_BASE(_n) + 0x18c)
++#define REG_FE_GDM_RX_ETH_L127_CNT_L(_n) (GDM_BASE(_n) + 0x190)
++#define REG_FE_GDM_RX_ETH_L255_CNT_L(_n) (GDM_BASE(_n) + 0x194)
++#define REG_FE_GDM_RX_ETH_L511_CNT_L(_n) (GDM_BASE(_n) + 0x198)
++#define REG_FE_GDM_RX_ETH_L1023_CNT_L(_n) (GDM_BASE(_n) + 0x19c)
++
++#define REG_PPE1_TB_HASH_CFG (PPE1_BASE + 0x250)
++#define PPE1_SRAM_TABLE_EN_MASK BIT(0)
++#define PPE1_SRAM_HASH1_EN_MASK BIT(8)
++#define PPE1_DRAM_TABLE_EN_MASK BIT(16)
++#define PPE1_DRAM_HASH1_EN_MASK BIT(24)
++
++#define REG_FE_GDM_TX_OK_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x280)
++#define REG_FE_GDM_TX_OK_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x284)
++#define REG_FE_GDM_TX_ETH_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x288)
++#define REG_FE_GDM_TX_ETH_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x28c)
++
++#define REG_FE_GDM_RX_OK_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x290)
++#define REG_FE_GDM_RX_OK_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x294)
++#define REG_FE_GDM_RX_ETH_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x298)
++#define REG_FE_GDM_RX_ETH_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x29c)
++#define REG_FE_GDM_TX_ETH_E64_CNT_H(_n) (GDM_BASE(_n) + 0x2b8)
++#define REG_FE_GDM_TX_ETH_L64_CNT_H(_n) (GDM_BASE(_n) + 0x2bc)
++#define REG_FE_GDM_TX_ETH_L127_CNT_H(_n) (GDM_BASE(_n) + 0x2c0)
++#define REG_FE_GDM_TX_ETH_L255_CNT_H(_n) (GDM_BASE(_n) + 0x2c4)
++#define REG_FE_GDM_TX_ETH_L511_CNT_H(_n) (GDM_BASE(_n) + 0x2c8)
++#define REG_FE_GDM_TX_ETH_L1023_CNT_H(_n) (GDM_BASE(_n) + 0x2cc)
++#define REG_FE_GDM_RX_ETH_E64_CNT_H(_n) (GDM_BASE(_n) + 0x2e8)
++#define REG_FE_GDM_RX_ETH_L64_CNT_H(_n) (GDM_BASE(_n) + 0x2ec)
++#define REG_FE_GDM_RX_ETH_L127_CNT_H(_n) (GDM_BASE(_n) + 0x2f0)
++#define REG_FE_GDM_RX_ETH_L255_CNT_H(_n) (GDM_BASE(_n) + 0x2f4)
++#define REG_FE_GDM_RX_ETH_L511_CNT_H(_n) (GDM_BASE(_n) + 0x2f8)
++#define REG_FE_GDM_RX_ETH_L1023_CNT_H(_n) (GDM_BASE(_n) + 0x2fc)
++
++#define REG_GDM2_CHN_RLS (GDM2_BASE + 0x20)
++#define MBI_RX_AGE_SEL_MASK GENMASK(26, 25)
++#define MBI_TX_AGE_SEL_MASK GENMASK(18, 17)
++
++#define REG_GDM3_FWD_CFG GDM3_BASE
++#define GDM3_PAD_EN_MASK BIT(28)
++
++#define REG_GDM4_FWD_CFG GDM4_BASE
++#define GDM4_PAD_EN_MASK BIT(28)
++#define GDM4_SPORT_OFFSET0_MASK GENMASK(11, 8)
++
++#define REG_GDM4_SRC_PORT_SET (GDM4_BASE + 0x23c)
++#define GDM4_SPORT_OFF2_MASK GENMASK(19, 16)
++#define GDM4_SPORT_OFF1_MASK GENMASK(15, 12)
++#define GDM4_SPORT_OFF0_MASK GENMASK(11, 8)
++
++#define REG_IP_FRAG_FP 0x2010
++#define IP_ASSEMBLE_PORT_MASK GENMASK(24, 21)
++#define IP_ASSEMBLE_NBQ_MASK GENMASK(20, 16)
++#define IP_FRAGMENT_PORT_MASK GENMASK(8, 5)
++#define IP_FRAGMENT_NBQ_MASK GENMASK(4, 0)
++
++#define REG_MC_VLAN_EN 0x2100
++#define MC_VLAN_EN_MASK BIT(0)
++
++#define REG_MC_VLAN_CFG 0x2104
++#define MC_VLAN_CFG_CMD_DONE_MASK BIT(31)
++#define MC_VLAN_CFG_TABLE_ID_MASK GENMASK(21, 16)
++#define MC_VLAN_CFG_PORT_ID_MASK GENMASK(11, 8)
++#define MC_VLAN_CFG_TABLE_SEL_MASK BIT(4)
++#define MC_VLAN_CFG_RW_MASK BIT(0)
++
++#define REG_MC_VLAN_DATA 0x2108
++
++#define REG_CDM5_RX_OQ1_DROP_CNT 0x29d4
++
++/* QDMA */
++#define REG_QDMA_GLOBAL_CFG 0x0004
++#define GLOBAL_CFG_RX_2B_OFFSET_MASK BIT(31)
++#define GLOBAL_CFG_DMA_PREFERENCE_MASK GENMASK(30, 29)
++#define GLOBAL_CFG_CPU_TXR_RR_MASK BIT(28)
++#define GLOBAL_CFG_DSCP_BYTE_SWAP_MASK BIT(27)
++#define GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK BIT(26)
++#define GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK BIT(25)
++#define GLOBAL_CFG_OAM_MODIFY_MASK BIT(24)
++#define GLOBAL_CFG_RESET_MASK BIT(23)
++#define GLOBAL_CFG_RESET_DONE_MASK BIT(22)
++#define GLOBAL_CFG_MULTICAST_EN_MASK BIT(21)
++#define GLOBAL_CFG_IRQ1_EN_MASK BIT(20)
++#define GLOBAL_CFG_IRQ0_EN_MASK BIT(19)
++#define GLOBAL_CFG_LOOPCNT_EN_MASK BIT(18)
++#define GLOBAL_CFG_RD_BYPASS_WR_MASK BIT(17)
++#define GLOBAL_CFG_QDMA_LOOPBACK_MASK BIT(16)
++#define GLOBAL_CFG_LPBK_RXQ_SEL_MASK GENMASK(13, 8)
++#define GLOBAL_CFG_CHECK_DONE_MASK BIT(7)
++#define GLOBAL_CFG_TX_WB_DONE_MASK BIT(6)
++#define GLOBAL_CFG_MAX_ISSUE_NUM_MASK GENMASK(5, 4)
++#define GLOBAL_CFG_RX_DMA_BUSY_MASK BIT(3)
++#define GLOBAL_CFG_RX_DMA_EN_MASK BIT(2)
++#define GLOBAL_CFG_TX_DMA_BUSY_MASK BIT(1)
++#define GLOBAL_CFG_TX_DMA_EN_MASK BIT(0)
++
++#define REG_FWD_DSCP_BASE 0x0010
++#define REG_FWD_BUF_BASE 0x0014
++
++#define REG_HW_FWD_DSCP_CFG 0x0018
++#define HW_FWD_DSCP_PAYLOAD_SIZE_MASK GENMASK(29, 28)
++#define HW_FWD_DSCP_SCATTER_LEN_MASK GENMASK(17, 16)
++#define HW_FWD_DSCP_MIN_SCATTER_LEN_MASK GENMASK(15, 0)
++
++#define REG_INT_STATUS(_n) \
++ (((_n) == 4) ? 0x0730 : \
++ ((_n) == 3) ? 0x0724 : \
++ ((_n) == 2) ? 0x0720 : \
++ ((_n) == 1) ? 0x0024 : 0x0020)
++
++#define REG_INT_ENABLE(_n) \
++ (((_n) == 4) ? 0x0750 : \
++ ((_n) == 3) ? 0x0744 : \
++ ((_n) == 2) ? 0x0740 : \
++ ((_n) == 1) ? 0x002c : 0x0028)
++
++/* QDMA_CSR_INT_ENABLE1 */
++#define RX15_COHERENT_INT_MASK BIT(31)
++#define RX14_COHERENT_INT_MASK BIT(30)
++#define RX13_COHERENT_INT_MASK BIT(29)
++#define RX12_COHERENT_INT_MASK BIT(28)
++#define RX11_COHERENT_INT_MASK BIT(27)
++#define RX10_COHERENT_INT_MASK BIT(26)
++#define RX9_COHERENT_INT_MASK BIT(25)
++#define RX8_COHERENT_INT_MASK BIT(24)
++#define RX7_COHERENT_INT_MASK BIT(23)
++#define RX6_COHERENT_INT_MASK BIT(22)
++#define RX5_COHERENT_INT_MASK BIT(21)
++#define RX4_COHERENT_INT_MASK BIT(20)
++#define RX3_COHERENT_INT_MASK BIT(19)
++#define RX2_COHERENT_INT_MASK BIT(18)
++#define RX1_COHERENT_INT_MASK BIT(17)
++#define RX0_COHERENT_INT_MASK BIT(16)
++#define TX7_COHERENT_INT_MASK BIT(15)
++#define TX6_COHERENT_INT_MASK BIT(14)
++#define TX5_COHERENT_INT_MASK BIT(13)
++#define TX4_COHERENT_INT_MASK BIT(12)
++#define TX3_COHERENT_INT_MASK BIT(11)
++#define TX2_COHERENT_INT_MASK BIT(10)
++#define TX1_COHERENT_INT_MASK BIT(9)
++#define TX0_COHERENT_INT_MASK BIT(8)
++#define CNT_OVER_FLOW_INT_MASK BIT(7)
++#define IRQ1_FULL_INT_MASK BIT(5)
++#define IRQ1_INT_MASK BIT(4)
++#define HWFWD_DSCP_LOW_INT_MASK BIT(3)
++#define HWFWD_DSCP_EMPTY_INT_MASK BIT(2)
++#define IRQ0_FULL_INT_MASK BIT(1)
++#define IRQ0_INT_MASK BIT(0)
++
++#define TX_DONE_INT_MASK(_n) \
++ ((_n) ? IRQ1_INT_MASK | IRQ1_FULL_INT_MASK \
++ : IRQ0_INT_MASK | IRQ0_FULL_INT_MASK)
++
++#define INT_TX_MASK \
++ (IRQ1_INT_MASK | IRQ1_FULL_INT_MASK | \
++ IRQ0_INT_MASK | IRQ0_FULL_INT_MASK)
++
++#define INT_IDX0_MASK \
++ (TX0_COHERENT_INT_MASK | TX1_COHERENT_INT_MASK | \
++ TX2_COHERENT_INT_MASK | TX3_COHERENT_INT_MASK | \
++ TX4_COHERENT_INT_MASK | TX5_COHERENT_INT_MASK | \
++ TX6_COHERENT_INT_MASK | TX7_COHERENT_INT_MASK | \
++ RX0_COHERENT_INT_MASK | RX1_COHERENT_INT_MASK | \
++ RX2_COHERENT_INT_MASK | RX3_COHERENT_INT_MASK | \
++ RX4_COHERENT_INT_MASK | RX7_COHERENT_INT_MASK | \
++ RX8_COHERENT_INT_MASK | RX9_COHERENT_INT_MASK | \
++ RX15_COHERENT_INT_MASK | INT_TX_MASK)
++
++/* QDMA_CSR_INT_ENABLE2 */
++#define RX15_NO_CPU_DSCP_INT_MASK BIT(31)
++#define RX14_NO_CPU_DSCP_INT_MASK BIT(30)
++#define RX13_NO_CPU_DSCP_INT_MASK BIT(29)
++#define RX12_NO_CPU_DSCP_INT_MASK BIT(28)
++#define RX11_NO_CPU_DSCP_INT_MASK BIT(27)
++#define RX10_NO_CPU_DSCP_INT_MASK BIT(26)
++#define RX9_NO_CPU_DSCP_INT_MASK BIT(25)
++#define RX8_NO_CPU_DSCP_INT_MASK BIT(24)
++#define RX7_NO_CPU_DSCP_INT_MASK BIT(23)
++#define RX6_NO_CPU_DSCP_INT_MASK BIT(22)
++#define RX5_NO_CPU_DSCP_INT_MASK BIT(21)
++#define RX4_NO_CPU_DSCP_INT_MASK BIT(20)
++#define RX3_NO_CPU_DSCP_INT_MASK BIT(19)
++#define RX2_NO_CPU_DSCP_INT_MASK BIT(18)
++#define RX1_NO_CPU_DSCP_INT_MASK BIT(17)
++#define RX0_NO_CPU_DSCP_INT_MASK BIT(16)
++#define RX15_DONE_INT_MASK BIT(15)
++#define RX14_DONE_INT_MASK BIT(14)
++#define RX13_DONE_INT_MASK BIT(13)
++#define RX12_DONE_INT_MASK BIT(12)
++#define RX11_DONE_INT_MASK BIT(11)
++#define RX10_DONE_INT_MASK BIT(10)
++#define RX9_DONE_INT_MASK BIT(9)
++#define RX8_DONE_INT_MASK BIT(8)
++#define RX7_DONE_INT_MASK BIT(7)
++#define RX6_DONE_INT_MASK BIT(6)
++#define RX5_DONE_INT_MASK BIT(5)
++#define RX4_DONE_INT_MASK BIT(4)
++#define RX3_DONE_INT_MASK BIT(3)
++#define RX2_DONE_INT_MASK BIT(2)
++#define RX1_DONE_INT_MASK BIT(1)
++#define RX0_DONE_INT_MASK BIT(0)
++
++#define RX_DONE_INT_MASK \
++ (RX0_DONE_INT_MASK | RX1_DONE_INT_MASK | \
++ RX2_DONE_INT_MASK | RX3_DONE_INT_MASK | \
++ RX4_DONE_INT_MASK | RX7_DONE_INT_MASK | \
++ RX8_DONE_INT_MASK | RX9_DONE_INT_MASK | \
++ RX15_DONE_INT_MASK)
++#define INT_IDX1_MASK \
++ (RX_DONE_INT_MASK | \
++ RX0_NO_CPU_DSCP_INT_MASK | RX1_NO_CPU_DSCP_INT_MASK | \
++ RX2_NO_CPU_DSCP_INT_MASK | RX3_NO_CPU_DSCP_INT_MASK | \
++ RX4_NO_CPU_DSCP_INT_MASK | RX7_NO_CPU_DSCP_INT_MASK | \
++ RX8_NO_CPU_DSCP_INT_MASK | RX9_NO_CPU_DSCP_INT_MASK | \
++ RX15_NO_CPU_DSCP_INT_MASK)
++
++/* QDMA_CSR_INT_ENABLE5 */
++#define TX31_COHERENT_INT_MASK BIT(31)
++#define TX30_COHERENT_INT_MASK BIT(30)
++#define TX29_COHERENT_INT_MASK BIT(29)
++#define TX28_COHERENT_INT_MASK BIT(28)
++#define TX27_COHERENT_INT_MASK BIT(27)
++#define TX26_COHERENT_INT_MASK BIT(26)
++#define TX25_COHERENT_INT_MASK BIT(25)
++#define TX24_COHERENT_INT_MASK BIT(24)
++#define TX23_COHERENT_INT_MASK BIT(23)
++#define TX22_COHERENT_INT_MASK BIT(22)
++#define TX21_COHERENT_INT_MASK BIT(21)
++#define TX20_COHERENT_INT_MASK BIT(20)
++#define TX19_COHERENT_INT_MASK BIT(19)
++#define TX18_COHERENT_INT_MASK BIT(18)
++#define TX17_COHERENT_INT_MASK BIT(17)
++#define TX16_COHERENT_INT_MASK BIT(16)
++#define TX15_COHERENT_INT_MASK BIT(15)
++#define TX14_COHERENT_INT_MASK BIT(14)
++#define TX13_COHERENT_INT_MASK BIT(13)
++#define TX12_COHERENT_INT_MASK BIT(12)
++#define TX11_COHERENT_INT_MASK BIT(11)
++#define TX10_COHERENT_INT_MASK BIT(10)
++#define TX9_COHERENT_INT_MASK BIT(9)
++#define TX8_COHERENT_INT_MASK BIT(8)
++
++#define INT_IDX4_MASK \
++ (TX8_COHERENT_INT_MASK | TX9_COHERENT_INT_MASK | \
++ TX10_COHERENT_INT_MASK | TX11_COHERENT_INT_MASK | \
++ TX12_COHERENT_INT_MASK | TX13_COHERENT_INT_MASK | \
++ TX14_COHERENT_INT_MASK | TX15_COHERENT_INT_MASK | \
++ TX16_COHERENT_INT_MASK | TX17_COHERENT_INT_MASK | \
++ TX18_COHERENT_INT_MASK | TX19_COHERENT_INT_MASK | \
++ TX20_COHERENT_INT_MASK | TX21_COHERENT_INT_MASK | \
++ TX22_COHERENT_INT_MASK | TX23_COHERENT_INT_MASK | \
++ TX24_COHERENT_INT_MASK | TX25_COHERENT_INT_MASK | \
++ TX26_COHERENT_INT_MASK | TX27_COHERENT_INT_MASK | \
++ TX28_COHERENT_INT_MASK | TX29_COHERENT_INT_MASK | \
++ TX30_COHERENT_INT_MASK | TX31_COHERENT_INT_MASK)
++
++#define REG_TX_IRQ_BASE(_n) ((_n) ? 0x0048 : 0x0050)
++
++#define REG_TX_IRQ_CFG(_n) ((_n) ? 0x004c : 0x0054)
++#define TX_IRQ_THR_MASK GENMASK(27, 16)
++#define TX_IRQ_DEPTH_MASK GENMASK(11, 0)
++
++#define REG_IRQ_CLEAR_LEN(_n) ((_n) ? 0x0064 : 0x0058)
++#define IRQ_CLEAR_LEN_MASK GENMASK(7, 0)
++
++#define REG_IRQ_STATUS(_n) ((_n) ? 0x0068 : 0x005c)
++#define IRQ_ENTRY_LEN_MASK GENMASK(27, 16)
++#define IRQ_HEAD_IDX_MASK GENMASK(11, 0)
++
++#define REG_TX_RING_BASE(_n) \
++ (((_n) < 8) ? 0x0100 + ((_n) << 5) : 0x0b00 + (((_n) - 8) << 5))
++
++#define REG_TX_RING_BLOCKING(_n) \
++ (((_n) < 8) ? 0x0104 + ((_n) << 5) : 0x0b04 + (((_n) - 8) << 5))
++
++#define TX_RING_IRQ_BLOCKING_MAP_MASK BIT(6)
++#define TX_RING_IRQ_BLOCKING_CFG_MASK BIT(4)
++#define TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK BIT(2)
++#define TX_RING_IRQ_BLOCKING_MAX_TH_TXRING_EN_MASK BIT(1)
++#define TX_RING_IRQ_BLOCKING_MIN_TH_TXRING_EN_MASK BIT(0)
++
++#define REG_TX_CPU_IDX(_n) \
++ (((_n) < 8) ? 0x0108 + ((_n) << 5) : 0x0b08 + (((_n) - 8) << 5))
++
++#define TX_RING_CPU_IDX_MASK GENMASK(15, 0)
++
++#define REG_TX_DMA_IDX(_n) \
++ (((_n) < 8) ? 0x010c + ((_n) << 5) : 0x0b0c + (((_n) - 8) << 5))
++
++#define TX_RING_DMA_IDX_MASK GENMASK(15, 0)
++
++#define IRQ_RING_IDX_MASK GENMASK(20, 16)
++#define IRQ_DESC_IDX_MASK GENMASK(15, 0)
++
++#define REG_RX_RING_BASE(_n) \
++ (((_n) < 16) ? 0x0200 + ((_n) << 5) : 0x0e00 + (((_n) - 16) << 5))
++
++#define REG_RX_RING_SIZE(_n) \
++ (((_n) < 16) ? 0x0204 + ((_n) << 5) : 0x0e04 + (((_n) - 16) << 5))
++
++#define RX_RING_THR_MASK GENMASK(31, 16)
++#define RX_RING_SIZE_MASK GENMASK(15, 0)
++
++#define REG_RX_CPU_IDX(_n) \
++ (((_n) < 16) ? 0x0208 + ((_n) << 5) : 0x0e08 + (((_n) - 16) << 5))
++
++#define RX_RING_CPU_IDX_MASK GENMASK(15, 0)
++
++#define REG_RX_DMA_IDX(_n) \
++ (((_n) < 16) ? 0x020c + ((_n) << 5) : 0x0e0c + (((_n) - 16) << 5))
++
++#define REG_RX_DELAY_INT_IDX(_n) \
++ (((_n) < 16) ? 0x0210 + ((_n) << 5) : 0x0e10 + (((_n) - 16) << 5))
++
++#define RX_DELAY_INT_MASK GENMASK(15, 0)
++
++#define RX_RING_DMA_IDX_MASK GENMASK(15, 0)
++
++#define REG_INGRESS_TRTCM_CFG 0x0070
++#define INGRESS_TRTCM_EN_MASK BIT(31)
++#define INGRESS_TRTCM_MODE_MASK BIT(30)
++#define INGRESS_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
++#define INGRESS_FAST_TICK_MASK GENMASK(15, 0)
++
++#define REG_QUEUE_CLOSE_CFG(_n) (0x00a0 + ((_n) & 0xfc))
++#define TXQ_DISABLE_CHAN_QUEUE_MASK(_n, _m) BIT((_m) + (((_n) & 0x3) << 3))
++
++#define REG_TXQ_DIS_CFG_BASE(_n) ((_n) ? 0x20a0 : 0x00a0)
++#define REG_TXQ_DIS_CFG(_n, _m) (REG_TXQ_DIS_CFG_BASE((_n)) + (_m) << 2)
++
++#define REG_CNTR_CFG(_n) (0x0400 + ((_n) << 3))
++#define CNTR_EN_MASK BIT(31)
++#define CNTR_ALL_CHAN_EN_MASK BIT(30)
++#define CNTR_ALL_QUEUE_EN_MASK BIT(29)
++#define CNTR_ALL_DSCP_RING_EN_MASK BIT(28)
++#define CNTR_SRC_MASK GENMASK(27, 24)
++#define CNTR_DSCP_RING_MASK GENMASK(20, 16)
++#define CNTR_CHAN_MASK GENMASK(7, 3)
++#define CNTR_QUEUE_MASK GENMASK(2, 0)
++
++#define REG_CNTR_VAL(_n) (0x0404 + ((_n) << 3))
++
++#define REG_LMGR_INIT_CFG 0x1000
++#define LMGR_INIT_START BIT(31)
++#define LMGR_SRAM_MODE_MASK BIT(30)
++#define HW_FWD_PKTSIZE_OVERHEAD_MASK GENMASK(27, 20)
++#define HW_FWD_DESC_NUM_MASK GENMASK(16, 0)
++
++#define REG_FWD_DSCP_LOW_THR 0x1004
++#define FWD_DSCP_LOW_THR_MASK GENMASK(17, 0)
++
++#define REG_EGRESS_RATE_METER_CFG 0x100c
++#define EGRESS_RATE_METER_EN_MASK BIT(29)
++#define EGRESS_RATE_METER_EQ_RATE_EN_MASK BIT(17)
++#define EGRESS_RATE_METER_WINDOW_SZ_MASK GENMASK(16, 12)
++#define EGRESS_RATE_METER_TIMESLICE_MASK GENMASK(10, 0)
++
++#define REG_EGRESS_TRTCM_CFG 0x1010
++#define EGRESS_TRTCM_EN_MASK BIT(31)
++#define EGRESS_TRTCM_MODE_MASK BIT(30)
++#define EGRESS_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
++#define EGRESS_FAST_TICK_MASK GENMASK(15, 0)
++
++#define REG_TXWRR_MODE_CFG 0x1020
++#define TWRR_WEIGHT_SCALE_MASK BIT(31)
++#define TWRR_WEIGHT_BASE_MASK BIT(3)
++
++#define REG_TXWRR_WEIGHT_CFG 0x1024
++#define TWRR_RW_CMD_MASK BIT(31)
++#define TWRR_RW_CMD_DONE BIT(30)
++#define TWRR_CHAN_IDX_MASK GENMASK(23, 19)
++#define TWRR_QUEUE_IDX_MASK GENMASK(18, 16)
++#define TWRR_VALUE_MASK GENMASK(15, 0)
++
++#define REG_PSE_BUF_USAGE_CFG 0x1028
++#define PSE_BUF_ESTIMATE_EN_MASK BIT(29)
++
++#define REG_CHAN_QOS_MODE(_n) (0x1040 + ((_n) << 2))
++#define CHAN_QOS_MODE_MASK(_n) GENMASK(2 + ((_n) << 2), (_n) << 2)
++
++#define REG_GLB_TRTCM_CFG 0x1080
++#define GLB_TRTCM_EN_MASK BIT(31)
++#define GLB_TRTCM_MODE_MASK BIT(30)
++#define GLB_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
++#define GLB_FAST_TICK_MASK GENMASK(15, 0)
++
++#define REG_TXQ_CNGST_CFG 0x10a0
++#define TXQ_CNGST_DROP_EN BIT(31)
++#define TXQ_CNGST_DEI_DROP_EN BIT(30)
++
++#define REG_SLA_TRTCM_CFG 0x1150
++#define SLA_TRTCM_EN_MASK BIT(31)
++#define SLA_TRTCM_MODE_MASK BIT(30)
++#define SLA_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
++#define SLA_FAST_TICK_MASK GENMASK(15, 0)
++
++/* CTRL */
++#define QDMA_DESC_DONE_MASK BIT(31)
++#define QDMA_DESC_DROP_MASK BIT(30) /* tx: drop - rx: overflow */
++#define QDMA_DESC_MORE_MASK BIT(29) /* more SG elements */
++#define QDMA_DESC_DEI_MASK BIT(25)
++#define QDMA_DESC_NO_DROP_MASK BIT(24)
++#define QDMA_DESC_LEN_MASK GENMASK(15, 0)
++/* DATA */
++#define QDMA_DESC_NEXT_ID_MASK GENMASK(15, 0)
++/* TX MSG0 */
++#define QDMA_ETH_TXMSG_MIC_IDX_MASK BIT(30)
++#define QDMA_ETH_TXMSG_SP_TAG_MASK GENMASK(29, 14)
++#define QDMA_ETH_TXMSG_ICO_MASK BIT(13)
++#define QDMA_ETH_TXMSG_UCO_MASK BIT(12)
++#define QDMA_ETH_TXMSG_TCO_MASK BIT(11)
++#define QDMA_ETH_TXMSG_TSO_MASK BIT(10)
++#define QDMA_ETH_TXMSG_FAST_MASK BIT(9)
++#define QDMA_ETH_TXMSG_OAM_MASK BIT(8)
++#define QDMA_ETH_TXMSG_CHAN_MASK GENMASK(7, 3)
++#define QDMA_ETH_TXMSG_QUEUE_MASK GENMASK(2, 0)
++/* TX MSG1 */
++#define QDMA_ETH_TXMSG_NO_DROP BIT(31)
++#define QDMA_ETH_TXMSG_METER_MASK GENMASK(30, 24) /* 0x7f no meters */
++#define QDMA_ETH_TXMSG_FPORT_MASK GENMASK(23, 20)
++#define QDMA_ETH_TXMSG_NBOQ_MASK GENMASK(19, 15)
++#define QDMA_ETH_TXMSG_HWF_MASK BIT(14)
++#define QDMA_ETH_TXMSG_HOP_MASK BIT(13)
++#define QDMA_ETH_TXMSG_PTP_MASK BIT(12)
++#define QDMA_ETH_TXMSG_ACNT_G1_MASK GENMASK(10, 6) /* 0x1f do not count */
++#define QDMA_ETH_TXMSG_ACNT_G0_MASK GENMASK(5, 0) /* 0x3f do not count */
++
++/* RX MSG1 */
++#define QDMA_ETH_RXMSG_DEI_MASK BIT(31)
++#define QDMA_ETH_RXMSG_IP6_MASK BIT(30)
++#define QDMA_ETH_RXMSG_IP4_MASK BIT(29)
++#define QDMA_ETH_RXMSG_IP4F_MASK BIT(28)
++#define QDMA_ETH_RXMSG_L4_VALID_MASK BIT(27)
++#define QDMA_ETH_RXMSG_L4F_MASK BIT(26)
++#define QDMA_ETH_RXMSG_SPORT_MASK GENMASK(25, 21)
++#define QDMA_ETH_RXMSG_CRSN_MASK GENMASK(20, 16)
++#define QDMA_ETH_RXMSG_PPE_ENTRY_MASK GENMASK(15, 0)
++
++struct airoha_qdma_desc {
++ __le32 rsv;
++ __le32 ctrl;
++ __le32 addr;
++ __le32 data;
++ __le32 msg0;
++ __le32 msg1;
++ __le32 msg2;
++ __le32 msg3;
++};
++
++/* CTRL0 */
++#define QDMA_FWD_DESC_CTX_MASK BIT(31)
++#define QDMA_FWD_DESC_RING_MASK GENMASK(30, 28)
++#define QDMA_FWD_DESC_IDX_MASK GENMASK(27, 16)
++#define QDMA_FWD_DESC_LEN_MASK GENMASK(15, 0)
++/* CTRL1 */
++#define QDMA_FWD_DESC_FIRST_IDX_MASK GENMASK(15, 0)
++/* CTRL2 */
++#define QDMA_FWD_DESC_MORE_PKT_NUM_MASK GENMASK(2, 0)
++
++struct airoha_qdma_fwd_desc {
++ __le32 addr;
++ __le32 ctrl0;
++ __le32 ctrl1;
++ __le32 ctrl2;
++ __le32 msg0;
++ __le32 msg1;
++ __le32 rsv0;
++ __le32 rsv1;
++};
++
++enum {
++ QDMA_INT_REG_IDX0,
++ QDMA_INT_REG_IDX1,
++ QDMA_INT_REG_IDX2,
++ QDMA_INT_REG_IDX3,
++ QDMA_INT_REG_IDX4,
++ QDMA_INT_REG_MAX
++};
++
++enum {
++ XSI_PCIE0_PORT,
++ XSI_PCIE1_PORT,
++ XSI_USB_PORT,
++ XSI_AE_PORT,
++ XSI_ETH_PORT,
++};
++
++enum {
++ XSI_PCIE0_VIP_PORT_MASK = BIT(22),
++ XSI_PCIE1_VIP_PORT_MASK = BIT(23),
++ XSI_USB_VIP_PORT_MASK = BIT(25),
++ XSI_ETH_VIP_PORT_MASK = BIT(24),
++};
++
++enum {
++ DEV_STATE_INITIALIZED,
++};
++
++enum {
++ CDM_CRSN_QSEL_Q1 = 1,
++ CDM_CRSN_QSEL_Q5 = 5,
++ CDM_CRSN_QSEL_Q6 = 6,
++ CDM_CRSN_QSEL_Q15 = 15,
++};
++
++enum {
++ CRSN_08 = 0x8,
++ CRSN_21 = 0x15, /* KA */
++ CRSN_22 = 0x16, /* hit bind and force route to CPU */
++ CRSN_24 = 0x18,
++ CRSN_25 = 0x19,
++};
++
++enum {
++ FE_PSE_PORT_CDM1,
++ FE_PSE_PORT_GDM1,
++ FE_PSE_PORT_GDM2,
++ FE_PSE_PORT_GDM3,
++ FE_PSE_PORT_PPE1,
++ FE_PSE_PORT_CDM2,
++ FE_PSE_PORT_CDM3,
++ FE_PSE_PORT_CDM4,
++ FE_PSE_PORT_PPE2,
++ FE_PSE_PORT_GDM4,
++ FE_PSE_PORT_CDM5,
++ FE_PSE_PORT_DROP = 0xf,
++};
++
++enum tx_sched_mode {
++ TC_SCH_WRR8,
++ TC_SCH_SP,
++ TC_SCH_WRR7,
++ TC_SCH_WRR6,
++ TC_SCH_WRR5,
++ TC_SCH_WRR4,
++ TC_SCH_WRR3,
++ TC_SCH_WRR2,
++};
++
++struct airoha_queue_entry {
++ union {
++ void *buf;
++ struct sk_buff *skb;
++ };
++ dma_addr_t dma_addr;
++ u16 dma_len;
++};
++
++struct airoha_queue {
++ struct airoha_qdma *qdma;
++
++ /* protect concurrent queue accesses */
++ spinlock_t lock;
++ struct airoha_queue_entry *entry;
++ struct airoha_qdma_desc *desc;
++ u16 head;
++ u16 tail;
++
++ int queued;
++ int ndesc;
++ int free_thr;
++ int buf_size;
++
++ struct napi_struct napi;
++ struct page_pool *page_pool;
++};
++
++struct airoha_tx_irq_queue {
++ struct airoha_qdma *qdma;
++
++ struct napi_struct napi;
++
++ int size;
++ u32 *q;
++};
++
++struct airoha_hw_stats {
++ /* protect concurrent hw_stats accesses */
++ spinlock_t lock;
++ struct u64_stats_sync syncp;
++
++ /* get_stats64 */
++ u64 rx_ok_pkts;
++ u64 tx_ok_pkts;
++ u64 rx_ok_bytes;
++ u64 tx_ok_bytes;
++ u64 rx_multicast;
++ u64 rx_errors;
++ u64 rx_drops;
++ u64 tx_drops;
++ u64 rx_crc_error;
++ u64 rx_over_errors;
++ /* ethtool stats */
++ u64 tx_broadcast;
++ u64 tx_multicast;
++ u64 tx_len[7];
++ u64 rx_broadcast;
++ u64 rx_fragment;
++ u64 rx_jabber;
++ u64 rx_len[7];
++};
++
++struct airoha_qdma {
++ struct airoha_eth *eth;
++ void __iomem *regs;
++
++ /* protect concurrent irqmask accesses */
++ spinlock_t irq_lock;
++ u32 irqmask[QDMA_INT_REG_MAX];
++ int irq;
++
++ struct airoha_tx_irq_queue q_tx_irq[AIROHA_NUM_TX_IRQ];
++
++ struct airoha_queue q_tx[AIROHA_NUM_TX_RING];
++ struct airoha_queue q_rx[AIROHA_NUM_RX_RING];
++
++ /* descriptor and packet buffers for qdma hw forward */
++ struct {
++ void *desc;
++ void *q;
++ } hfwd;
++};
++
++struct airoha_gdm_port {
++ struct airoha_qdma *qdma;
++ struct net_device *dev;
++ int id;
++
++ struct airoha_hw_stats stats;
++
++ /* qos stats counters */
++ u64 cpu_tx_packets;
++ u64 fwd_tx_packets;
++};
++
++struct airoha_eth {
++ struct device *dev;
++
++ unsigned long state;
++ void __iomem *fe_regs;
++
++ struct reset_control_bulk_data rsts[AIROHA_MAX_NUM_RSTS];
++ struct reset_control_bulk_data xsi_rsts[AIROHA_MAX_NUM_XSI_RSTS];
++
++ struct net_device *napi_dev;
++
++ struct airoha_qdma qdma[AIROHA_MAX_NUM_QDMA];
++ struct airoha_gdm_port *ports[AIROHA_MAX_NUM_GDM_PORTS];
++};
++
++static u32 airoha_rr(void __iomem *base, u32 offset)
++{
++ return readl(base + offset);
++}
++
++static void airoha_wr(void __iomem *base, u32 offset, u32 val)
++{
++ writel(val, base + offset);
++}
++
++static u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val)
++{
++ val |= (airoha_rr(base, offset) & ~mask);
++ airoha_wr(base, offset, val);
++
++ return val;
++}
++
++#define airoha_fe_rr(eth, offset) \
++ airoha_rr((eth)->fe_regs, (offset))
++#define airoha_fe_wr(eth, offset, val) \
++ airoha_wr((eth)->fe_regs, (offset), (val))
++#define airoha_fe_rmw(eth, offset, mask, val) \
++ airoha_rmw((eth)->fe_regs, (offset), (mask), (val))
++#define airoha_fe_set(eth, offset, val) \
++ airoha_rmw((eth)->fe_regs, (offset), 0, (val))
++#define airoha_fe_clear(eth, offset, val) \
++ airoha_rmw((eth)->fe_regs, (offset), (val), 0)
++
++#define airoha_qdma_rr(qdma, offset) \
++ airoha_rr((qdma)->regs, (offset))
++#define airoha_qdma_wr(qdma, offset, val) \
++ airoha_wr((qdma)->regs, (offset), (val))
++#define airoha_qdma_rmw(qdma, offset, mask, val) \
++ airoha_rmw((qdma)->regs, (offset), (mask), (val))
++#define airoha_qdma_set(qdma, offset, val) \
++ airoha_rmw((qdma)->regs, (offset), 0, (val))
++#define airoha_qdma_clear(qdma, offset, val) \
++ airoha_rmw((qdma)->regs, (offset), (val), 0)
++
++static void airoha_qdma_set_irqmask(struct airoha_qdma *qdma, int index,
++ u32 clear, u32 set)
++{
++ unsigned long flags;
++
++ if (WARN_ON_ONCE(index >= ARRAY_SIZE(qdma->irqmask)))
++ return;
++
++ spin_lock_irqsave(&qdma->irq_lock, flags);
++
++ qdma->irqmask[index] &= ~clear;
++ qdma->irqmask[index] |= set;
++ airoha_qdma_wr(qdma, REG_INT_ENABLE(index), qdma->irqmask[index]);
++ /* Read irq_enable register in order to guarantee the update above
++ * completes in the spinlock critical section.
++ */
++ airoha_qdma_rr(qdma, REG_INT_ENABLE(index));
++
++ spin_unlock_irqrestore(&qdma->irq_lock, flags);
++}
++
++static void airoha_qdma_irq_enable(struct airoha_qdma *qdma, int index,
++ u32 mask)
++{
++ airoha_qdma_set_irqmask(qdma, index, 0, mask);
++}
++
++static void airoha_qdma_irq_disable(struct airoha_qdma *qdma, int index,
++ u32 mask)
++{
++ airoha_qdma_set_irqmask(qdma, index, mask, 0);
++}
++
++static bool airhoa_is_lan_gdm_port(struct airoha_gdm_port *port)
++{
++ /* GDM1 port on EN7581 SoC is connected to the lan dsa switch.
++ * GDM{2,3,4} can be used as wan port connected to an external
++ * phy module.
++ */
++ return port->id == 1;
++}
++
++static void airoha_set_macaddr(struct airoha_gdm_port *port, const u8 *addr)
++{
++ struct airoha_eth *eth = port->qdma->eth;
++ u32 val, reg;
++
++ reg = airhoa_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H
++ : REG_FE_WAN_MAC_H;
++ val = (addr[0] << 16) | (addr[1] << 8) | addr[2];
++ airoha_fe_wr(eth, reg, val);
++
++ val = (addr[3] << 16) | (addr[4] << 8) | addr[5];
++ airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), val);
++ airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), val);
++}
++
++static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr,
++ u32 val)
++{
++ airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK,
++ FIELD_PREP(GDM_OCFQ_MASK, val));
++ airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK,
++ FIELD_PREP(GDM_MCFQ_MASK, val));
++ airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK,
++ FIELD_PREP(GDM_BCFQ_MASK, val));
++ airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK,
++ FIELD_PREP(GDM_UCFQ_MASK, val));
++}
++
++static int airoha_set_gdm_port(struct airoha_eth *eth, int port, bool enable)
++{
++ u32 val = enable ? FE_PSE_PORT_PPE1 : FE_PSE_PORT_DROP;
++ u32 vip_port, cfg_addr;
++
++ switch (port) {
++ case XSI_PCIE0_PORT:
++ vip_port = XSI_PCIE0_VIP_PORT_MASK;
++ cfg_addr = REG_GDM_FWD_CFG(3);
++ break;
++ case XSI_PCIE1_PORT:
++ vip_port = XSI_PCIE1_VIP_PORT_MASK;
++ cfg_addr = REG_GDM_FWD_CFG(3);
++ break;
++ case XSI_USB_PORT:
++ vip_port = XSI_USB_VIP_PORT_MASK;
++ cfg_addr = REG_GDM_FWD_CFG(4);
++ break;
++ case XSI_ETH_PORT:
++ vip_port = XSI_ETH_VIP_PORT_MASK;
++ cfg_addr = REG_GDM_FWD_CFG(4);
++ break;
++ default:
++ return -EINVAL;
++ }
++
++ if (enable) {
++ airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port);
++ airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port);
++ } else {
++ airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port);
++ airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port);
++ }
++
++ airoha_set_gdm_port_fwd_cfg(eth, cfg_addr, val);
++
++ return 0;
++}
++
++static int airoha_set_gdm_ports(struct airoha_eth *eth, bool enable)
++{
++ const int port_list[] = {
++ XSI_PCIE0_PORT,
++ XSI_PCIE1_PORT,
++ XSI_USB_PORT,
++ XSI_ETH_PORT
++ };
++ int i, err;
++
++ for (i = 0; i < ARRAY_SIZE(port_list); i++) {
++ err = airoha_set_gdm_port(eth, port_list[i], enable);
++ if (err)
++ goto error;
++ }
++
++ return 0;
++
++error:
++ for (i--; i >= 0; i--)
++ airoha_set_gdm_port(eth, port_list[i], false);
++
++ return err;
++}
++
++static void airoha_fe_maccr_init(struct airoha_eth *eth)
++{
++ int p;
++
++ for (p = 1; p <= ARRAY_SIZE(eth->ports); p++) {
++ airoha_fe_set(eth, REG_GDM_FWD_CFG(p),
++ GDM_TCP_CKSUM | GDM_UDP_CKSUM | GDM_IP4_CKSUM |
++ GDM_DROP_CRC_ERR);
++ airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(p),
++ FE_PSE_PORT_CDM1);
++ airoha_fe_rmw(eth, REG_GDM_LEN_CFG(p),
++ GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK,
++ FIELD_PREP(GDM_SHORT_LEN_MASK, 60) |
++ FIELD_PREP(GDM_LONG_LEN_MASK, 4004));
++ }
++
++ airoha_fe_rmw(eth, REG_CDM1_VLAN_CTRL, CDM1_VLAN_MASK,
++ FIELD_PREP(CDM1_VLAN_MASK, 0x8100));
++
++ airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD);
++}
++
++static void airoha_fe_vip_setup(struct airoha_eth *eth)
++{
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK);
++
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(4),
++ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
++ PATN_EN_MASK);
++
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(6),
++ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
++ PATN_EN_MASK);
++
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(7),
++ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
++ PATN_EN_MASK);
++
++ /* BOOTP (0x43) */
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(8),
++ PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
++ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
++
++ /* BOOTP (0x44) */
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(9),
++ PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
++ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
++
++ /* ISAKMP */
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(10),
++ PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
++ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
++
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(11),
++ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
++ PATN_EN_MASK);
++
++ /* DHCPv6 */
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(12),
++ PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
++ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
++
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(19),
++ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
++ PATN_EN_MASK);
++
++ /* ETH->ETH_P_1905 (0x893a) */
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(20),
++ PATN_FCPU_EN_MASK | PATN_EN_MASK);
++
++ airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP);
++ airoha_fe_wr(eth, REG_FE_VIP_EN(21),
++ PATN_FCPU_EN_MASK | PATN_EN_MASK);
++}
++
++static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth,
++ u32 port, u32 queue)
++{
++ u32 val;
++
++ airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
++ PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK,
++ FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
++ FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue));
++ val = airoha_fe_rr(eth, REG_FE_PSE_QUEUE_CFG_VAL);
++
++ return FIELD_GET(PSE_CFG_OQ_RSV_MASK, val);
++}
++
++static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth,
++ u32 port, u32 queue, u32 val)
++{
++ airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK,
++ FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val));
++ airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
++ PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK |
++ PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK,
++ FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
++ FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) |
++ PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK);
++}
++
++static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth)
++{
++ u32 val = airoha_fe_rr(eth, REG_FE_PSE_BUF_SET);
++
++ return FIELD_GET(PSE_ALLRSV_MASK, val);
++}
++
++static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth,
++ u32 port, u32 queue, u32 val)
++{
++ u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue);
++ u32 tmp, all_rsv, fq_limit;
++
++ airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val);
++
++ /* modify all rsv */
++ all_rsv = airoha_fe_get_pse_all_rsv(eth);
++ all_rsv += (val - orig_val);
++ airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
++ FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
++
++ /* modify hthd */
++ tmp = airoha_fe_rr(eth, PSE_FQ_CFG);
++ fq_limit = FIELD_GET(PSE_FQ_LIMIT_MASK, tmp);
++ tmp = fq_limit - all_rsv - 0x20;
++ airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
++ PSE_SHARE_USED_HTHD_MASK,
++ FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp));
++
++ tmp = fq_limit - all_rsv - 0x100;
++ airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
++ PSE_SHARE_USED_MTHD_MASK,
++ FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp));
++ tmp = (3 * tmp) >> 2;
++ airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET,
++ PSE_SHARE_USED_LTHD_MASK,
++ FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp));
++
++ return 0;
++}
++
++static void airoha_fe_pse_ports_init(struct airoha_eth *eth)
++{
++ const u32 pse_port_num_queues[] = {
++ [FE_PSE_PORT_CDM1] = 6,
++ [FE_PSE_PORT_GDM1] = 6,
++ [FE_PSE_PORT_GDM2] = 32,
++ [FE_PSE_PORT_GDM3] = 6,
++ [FE_PSE_PORT_PPE1] = 4,
++ [FE_PSE_PORT_CDM2] = 6,
++ [FE_PSE_PORT_CDM3] = 8,
++ [FE_PSE_PORT_CDM4] = 10,
++ [FE_PSE_PORT_PPE2] = 4,
++ [FE_PSE_PORT_GDM4] = 2,
++ [FE_PSE_PORT_CDM5] = 2,
++ };
++ u32 all_rsv;
++ int q;
++
++ all_rsv = airoha_fe_get_pse_all_rsv(eth);
++ /* hw misses PPE2 oq rsv */
++ all_rsv += PSE_RSV_PAGES * pse_port_num_queues[FE_PSE_PORT_PPE2];
++ airoha_fe_set(eth, REG_FE_PSE_BUF_SET, all_rsv);
++
++ /* CMD1 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q,
++ PSE_QUEUE_RSV_PAGES);
++ /* GMD1 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q,
++ PSE_QUEUE_RSV_PAGES);
++ /* GMD2 */
++ for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0);
++ /* GMD3 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q,
++ PSE_QUEUE_RSV_PAGES);
++ /* PPE1 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) {
++ if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q,
++ PSE_QUEUE_RSV_PAGES);
++ else
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0);
++ }
++ /* CDM2 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q,
++ PSE_QUEUE_RSV_PAGES);
++ /* CDM3 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0);
++ /* CDM4 */
++ for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q,
++ PSE_QUEUE_RSV_PAGES);
++ /* PPE2 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) {
++ if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, q,
++ PSE_QUEUE_RSV_PAGES);
++ else
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, q, 0);
++ }
++ /* GMD4 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q,
++ PSE_QUEUE_RSV_PAGES);
++ /* CDM5 */
++ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++)
++ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q,
++ PSE_QUEUE_RSV_PAGES);
++}
++
++static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth)
++{
++ int i;
++
++ for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) {
++ int err, j;
++ u32 val;
++
++ airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
++
++ val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
++ MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK;
++ airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
++ err = read_poll_timeout(airoha_fe_rr, val,
++ val & MC_VLAN_CFG_CMD_DONE_MASK,
++ USEC_PER_MSEC, 5 * USEC_PER_MSEC,
++ false, eth, REG_MC_VLAN_CFG);
++ if (err)
++ return err;
++
++ for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) {
++ airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
++
++ val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
++ FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) |
++ MC_VLAN_CFG_RW_MASK;
++ airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
++ err = read_poll_timeout(airoha_fe_rr, val,
++ val & MC_VLAN_CFG_CMD_DONE_MASK,
++ USEC_PER_MSEC,
++ 5 * USEC_PER_MSEC, false, eth,
++ REG_MC_VLAN_CFG);
++ if (err)
++ return err;
++ }
++ }
++
++ return 0;
++}
++
++static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth)
++{
++ /* CDM1_CRSN_QSEL */
++ airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_22 >> 2),
++ CDM1_CRSN_QSEL_REASON_MASK(CRSN_22),
++ FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_22),
++ CDM_CRSN_QSEL_Q1));
++ airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_08 >> 2),
++ CDM1_CRSN_QSEL_REASON_MASK(CRSN_08),
++ FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_08),
++ CDM_CRSN_QSEL_Q1));
++ airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_21 >> 2),
++ CDM1_CRSN_QSEL_REASON_MASK(CRSN_21),
++ FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_21),
++ CDM_CRSN_QSEL_Q1));
++ airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_24 >> 2),
++ CDM1_CRSN_QSEL_REASON_MASK(CRSN_24),
++ FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_24),
++ CDM_CRSN_QSEL_Q6));
++ airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_25 >> 2),
++ CDM1_CRSN_QSEL_REASON_MASK(CRSN_25),
++ FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_25),
++ CDM_CRSN_QSEL_Q1));
++ /* CDM2_CRSN_QSEL */
++ airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_08 >> 2),
++ CDM2_CRSN_QSEL_REASON_MASK(CRSN_08),
++ FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_08),
++ CDM_CRSN_QSEL_Q1));
++ airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_21 >> 2),
++ CDM2_CRSN_QSEL_REASON_MASK(CRSN_21),
++ FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_21),
++ CDM_CRSN_QSEL_Q1));
++ airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_22 >> 2),
++ CDM2_CRSN_QSEL_REASON_MASK(CRSN_22),
++ FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_22),
++ CDM_CRSN_QSEL_Q1));
++ airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_24 >> 2),
++ CDM2_CRSN_QSEL_REASON_MASK(CRSN_24),
++ FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_24),
++ CDM_CRSN_QSEL_Q6));
++ airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_25 >> 2),
++ CDM2_CRSN_QSEL_REASON_MASK(CRSN_25),
++ FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_25),
++ CDM_CRSN_QSEL_Q1));
++}
++
++static int airoha_fe_init(struct airoha_eth *eth)
++{
++ airoha_fe_maccr_init(eth);
++
++ /* PSE IQ reserve */
++ airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK,
++ FIELD_PREP(PSE_IQ_RES1_P2_MASK, 0x10));
++ airoha_fe_rmw(eth, REG_PSE_IQ_REV2,
++ PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK,
++ FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) |
++ FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34));
++
++ /* enable FE copy engine for MC/KA/DPI */
++ airoha_fe_wr(eth, REG_FE_PCE_CFG,
++ PCE_DPI_EN_MASK | PCE_KA_EN_MASK | PCE_MC_EN_MASK);
++ /* set vip queue selection to ring 1 */
++ airoha_fe_rmw(eth, REG_CDM1_FWD_CFG, CDM1_VIP_QSEL_MASK,
++ FIELD_PREP(CDM1_VIP_QSEL_MASK, 0x4));
++ airoha_fe_rmw(eth, REG_CDM2_FWD_CFG, CDM2_VIP_QSEL_MASK,
++ FIELD_PREP(CDM2_VIP_QSEL_MASK, 0x4));
++ /* set GDM4 source interface offset to 8 */
++ airoha_fe_rmw(eth, REG_GDM4_SRC_PORT_SET,
++ GDM4_SPORT_OFF2_MASK |
++ GDM4_SPORT_OFF1_MASK |
++ GDM4_SPORT_OFF0_MASK,
++ FIELD_PREP(GDM4_SPORT_OFF2_MASK, 8) |
++ FIELD_PREP(GDM4_SPORT_OFF1_MASK, 8) |
++ FIELD_PREP(GDM4_SPORT_OFF0_MASK, 8));
++
++ /* set PSE Page as 128B */
++ airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG,
++ FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK,
++ FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, 2) |
++ FE_DMA_GLO_PG_SZ_MASK);
++ airoha_fe_wr(eth, REG_FE_RST_GLO_CFG,
++ FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK |
++ FE_RST_GDM4_MBI_ARB_MASK);
++ usleep_range(1000, 2000);
++
++ /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1
++ * connect other rings to PSE Port0 OQ-0
++ */
++ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4));
++ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28));
++ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4));
++ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28));
++
++ airoha_fe_vip_setup(eth);
++ airoha_fe_pse_ports_init(eth);
++
++ airoha_fe_set(eth, REG_GDM_MISC_CFG,
++ GDM2_RDM_ACK_WAIT_PREF_MASK |
++ GDM2_CHN_VLD_MODE_MASK);
++ airoha_fe_rmw(eth, REG_CDM2_FWD_CFG, CDM2_OAM_QSEL_MASK, 15);
++
++ /* init fragment and assemble Force Port */
++ /* NPU Core-3, NPU Bridge Channel-3 */
++ airoha_fe_rmw(eth, REG_IP_FRAG_FP,
++ IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK,
++ FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) |
++ FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3));
++ /* QDMA LAN, RX Ring-22 */
++ airoha_fe_rmw(eth, REG_IP_FRAG_FP,
++ IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK,
++ FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) |
++ FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22));
++
++ airoha_fe_set(eth, REG_GDM3_FWD_CFG, GDM3_PAD_EN_MASK);
++ airoha_fe_set(eth, REG_GDM4_FWD_CFG, GDM4_PAD_EN_MASK);
++
++ airoha_fe_crsn_qsel_init(eth);
++
++ airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK);
++ airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK);
++
++ /* default aging mode for mbi unlock issue */
++ airoha_fe_rmw(eth, REG_GDM2_CHN_RLS,
++ MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK,
++ FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) |
++ FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3));
++
++ /* disable IFC by default */
++ airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK);
++
++ /* enable 1:N vlan action, init vlan table */
++ airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK);
++
++ return airoha_fe_mc_vlan_clear(eth);
++}
++
++static int airoha_qdma_fill_rx_queue(struct airoha_queue *q)
++{
++ enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
++ struct airoha_qdma *qdma = q->qdma;
++ struct airoha_eth *eth = qdma->eth;
++ int qid = q - &qdma->q_rx[0];
++ int nframes = 0;
++
++ while (q->queued < q->ndesc - 1) {
++ struct airoha_queue_entry *e = &q->entry[q->head];
++ struct airoha_qdma_desc *desc = &q->desc[q->head];
++ struct page *page;
++ int offset;
++ u32 val;
++
++ page = page_pool_dev_alloc_frag(q->page_pool, &offset,
++ q->buf_size);
++ if (!page)
++ break;
++
++ q->head = (q->head + 1) % q->ndesc;
++ q->queued++;
++ nframes++;
++
++ e->buf = page_address(page) + offset;
++ e->dma_addr = page_pool_get_dma_addr(page) + offset;
++ e->dma_len = SKB_WITH_OVERHEAD(q->buf_size);
++
++ dma_sync_single_for_device(eth->dev, e->dma_addr, e->dma_len,
++ dir);
++
++ val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len);
++ WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
++ WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr));
++ val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head);
++ WRITE_ONCE(desc->data, cpu_to_le32(val));
++ WRITE_ONCE(desc->msg0, 0);
++ WRITE_ONCE(desc->msg1, 0);
++ WRITE_ONCE(desc->msg2, 0);
++ WRITE_ONCE(desc->msg3, 0);
++
++ airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid),
++ RX_RING_CPU_IDX_MASK,
++ FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
++ }
++
++ return nframes;
++}
++
++static int airoha_qdma_get_gdm_port(struct airoha_eth *eth,
++ struct airoha_qdma_desc *desc)
++{
++ u32 port, sport, msg1 = le32_to_cpu(desc->msg1);
++
++ sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, msg1);
++ switch (sport) {
++ case 0x10 ... 0x13:
++ port = 0;
++ break;
++ case 0x2 ... 0x4:
++ port = sport - 1;
++ break;
++ default:
++ return -EINVAL;
++ }
++
++ return port >= ARRAY_SIZE(eth->ports) ? -EINVAL : port;
++}
++
++static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
++{
++ enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
++ struct airoha_qdma *qdma = q->qdma;
++ struct airoha_eth *eth = qdma->eth;
++ int qid = q - &qdma->q_rx[0];
++ int done = 0;
++
++ while (done < budget) {
++ struct airoha_queue_entry *e = &q->entry[q->tail];
++ struct airoha_qdma_desc *desc = &q->desc[q->tail];
++ dma_addr_t dma_addr = le32_to_cpu(desc->addr);
++ u32 desc_ctrl = le32_to_cpu(desc->ctrl);
++ struct sk_buff *skb;
++ int len, p;
++
++ if (!(desc_ctrl & QDMA_DESC_DONE_MASK))
++ break;
++
++ if (!dma_addr)
++ break;
++
++ len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
++ if (!len)
++ break;
++
++ q->tail = (q->tail + 1) % q->ndesc;
++ q->queued--;
++
++ dma_sync_single_for_cpu(eth->dev, dma_addr,
++ SKB_WITH_OVERHEAD(q->buf_size), dir);
++
++ p = airoha_qdma_get_gdm_port(eth, desc);
++ if (p < 0 || !eth->ports[p]) {
++ page_pool_put_full_page(q->page_pool,
++ virt_to_head_page(e->buf),
++ true);
++ continue;
++ }
++
++ skb = napi_build_skb(e->buf, q->buf_size);
++ if (!skb) {
++ page_pool_put_full_page(q->page_pool,
++ virt_to_head_page(e->buf),
++ true);
++ break;
++ }
++
++ skb_reserve(skb, 2);
++ __skb_put(skb, len);
++ skb_mark_for_recycle(skb);
++ skb->dev = eth->ports[p]->dev;
++ skb->protocol = eth_type_trans(skb, skb->dev);
++ skb->ip_summed = CHECKSUM_UNNECESSARY;
++ skb_record_rx_queue(skb, qid);
++ napi_gro_receive(&q->napi, skb);
++
++ done++;
++ }
++ airoha_qdma_fill_rx_queue(q);
++
++ return done;
++}
++
++static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
++{
++ struct airoha_queue *q = container_of(napi, struct airoha_queue, napi);
++ int cur, done = 0;
++
++ do {
++ cur = airoha_qdma_rx_process(q, budget - done);
++ done += cur;
++ } while (cur && done < budget);
++
++ if (done < budget && napi_complete(napi))
++ airoha_qdma_irq_enable(q->qdma, QDMA_INT_REG_IDX1,
++ RX_DONE_INT_MASK);
++
++ return done;
++}
++
++static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
++ struct airoha_qdma *qdma, int ndesc)
++{
++ const struct page_pool_params pp_params = {
++ .order = 0,
++ .pool_size = 256,
++ .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
++ .dma_dir = DMA_FROM_DEVICE,
++ .max_len = PAGE_SIZE,
++ .nid = NUMA_NO_NODE,
++ .dev = qdma->eth->dev,
++ .napi = &q->napi,
++ };
++ struct airoha_eth *eth = qdma->eth;
++ int qid = q - &qdma->q_rx[0], thr;
++ dma_addr_t dma_addr;
++
++ q->buf_size = PAGE_SIZE / 2;
++ q->qdma = qdma;
++
++ q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
++ GFP_KERNEL);
++ if (!q->entry)
++ return -ENOMEM;
++
++ q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc),
++ &dma_addr, GFP_KERNEL);
++ if (!q->desc)
++ return -ENOMEM;
++
++ q->page_pool = page_pool_create(&pp_params);
++ if (IS_ERR(q->page_pool)) {
++ int err = PTR_ERR(q->page_pool);
++
++ q->page_pool = NULL;
++ return err;
++ }
++
++ q->ndesc = ndesc;
++ netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
++
++ airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);
++ airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),
++ RX_RING_SIZE_MASK,
++ FIELD_PREP(RX_RING_SIZE_MASK, ndesc));
++
++ thr = clamp(ndesc >> 3, 1, 32);
++ airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK,
++ FIELD_PREP(RX_RING_THR_MASK, thr));
++ airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
++ FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head));
++
++ airoha_qdma_fill_rx_queue(q);
++
++ return 0;
++}
++
++static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
++{
++ struct airoha_qdma *qdma = q->qdma;
++ struct airoha_eth *eth = qdma->eth;
++ int qid = q - &qdma->q_rx[0];
++
++ while (q->queued) {
++ struct airoha_queue_entry *e = &q->entry[q->tail];
++ struct airoha_qdma_desc *desc = &q->desc[q->tail];
++ struct page *page = virt_to_head_page(e->buf);
++
++ dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
++ page_pool_get_dma_dir(q->page_pool));
++ page_pool_put_full_page(q->page_pool, page, false);
++ /* Reset DMA descriptor */
++ WRITE_ONCE(desc->ctrl, 0);
++ WRITE_ONCE(desc->addr, 0);
++ WRITE_ONCE(desc->data, 0);
++ WRITE_ONCE(desc->msg0, 0);
++ WRITE_ONCE(desc->msg1, 0);
++ WRITE_ONCE(desc->msg2, 0);
++ WRITE_ONCE(desc->msg3, 0);
++
++ q->tail = (q->tail + 1) % q->ndesc;
++ q->queued--;
++ }
++
++ q->head = q->tail;
++ /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is
++ * empty.
++ */
++ airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK,
++ FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
++ airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
++ FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail));
++}
++
++static int airoha_qdma_init_rx(struct airoha_qdma *qdma)
++{
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
++ int err;
++
++ if (!(RX_DONE_INT_MASK & BIT(i))) {
++ /* rx-queue not binded to irq */
++ continue;
++ }
++
++ err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma,
++ RX_DSCP_NUM(i));
++ if (err)
++ return err;
++ }
++
++ return 0;
++}
++
++static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
++{
++ struct airoha_tx_irq_queue *irq_q;
++ int id, done = 0, irq_queued;
++ struct airoha_qdma *qdma;
++ struct airoha_eth *eth;
++ u32 status, head;
++
++ irq_q = container_of(napi, struct airoha_tx_irq_queue, napi);
++ qdma = irq_q->qdma;
++ id = irq_q - &qdma->q_tx_irq[0];
++ eth = qdma->eth;
++
++ status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id));
++ head = FIELD_GET(IRQ_HEAD_IDX_MASK, status);
++ head = head % irq_q->size;
++ irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status);
++
++ while (irq_queued > 0 && done < budget) {
++ u32 qid, last, val = irq_q->q[head];
++ struct airoha_queue *q;
++
++ if (val == 0xff)
++ break;
++
++ irq_q->q[head] = 0xff; /* mark as done */
++ head = (head + 1) % irq_q->size;
++ irq_queued--;
++ done++;
++
++ last = FIELD_GET(IRQ_DESC_IDX_MASK, val);
++ qid = FIELD_GET(IRQ_RING_IDX_MASK, val);
++
++ if (qid >= ARRAY_SIZE(qdma->q_tx))
++ continue;
++
++ q = &qdma->q_tx[qid];
++ if (!q->ndesc)
++ continue;
++
++ spin_lock_bh(&q->lock);
++
++ while (q->queued > 0) {
++ struct airoha_qdma_desc *desc = &q->desc[q->tail];
++ struct airoha_queue_entry *e = &q->entry[q->tail];
++ u32 desc_ctrl = le32_to_cpu(desc->ctrl);
++ struct sk_buff *skb = e->skb;
++ u16 index = q->tail;
++
++ if (!(desc_ctrl & QDMA_DESC_DONE_MASK) &&
++ !(desc_ctrl & QDMA_DESC_DROP_MASK))
++ break;
++
++ q->tail = (q->tail + 1) % q->ndesc;
++ q->queued--;
++
++ dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
++ DMA_TO_DEVICE);
++
++ WRITE_ONCE(desc->msg0, 0);
++ WRITE_ONCE(desc->msg1, 0);
++
++ if (skb) {
++ u16 queue = skb_get_queue_mapping(skb);
++ struct netdev_queue *txq;
++
++ txq = netdev_get_tx_queue(skb->dev, queue);
++ netdev_tx_completed_queue(txq, 1, skb->len);
++ if (netif_tx_queue_stopped(txq) &&
++ q->ndesc - q->queued >= q->free_thr)
++ netif_tx_wake_queue(txq);
++
++ dev_kfree_skb_any(skb);
++ e->skb = NULL;
++ }
++
++ if (index == last)
++ break;
++ }
++
++ spin_unlock_bh(&q->lock);
++ }
++
++ if (done) {
++ int i, len = done >> 7;
++
++ for (i = 0; i < len; i++)
++ airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
++ IRQ_CLEAR_LEN_MASK, 0x80);
++ airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
++ IRQ_CLEAR_LEN_MASK, (done & 0x7f));
++ }
++
++ if (done < budget && napi_complete(napi))
++ airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX0,
++ TX_DONE_INT_MASK(id));
++
++ return done;
++}
++
++static int airoha_qdma_init_tx_queue(struct airoha_queue *q,
++ struct airoha_qdma *qdma, int size)
++{
++ struct airoha_eth *eth = qdma->eth;
++ int i, qid = q - &qdma->q_tx[0];
++ dma_addr_t dma_addr;
++
++ spin_lock_init(&q->lock);
++ q->ndesc = size;
++ q->qdma = qdma;
++ q->free_thr = 1 + MAX_SKB_FRAGS;
++
++ q->entry = devm_kzalloc(eth->dev, q->ndesc * sizeof(*q->entry),
++ GFP_KERNEL);
++ if (!q->entry)
++ return -ENOMEM;
++
++ q->desc = dmam_alloc_coherent(eth->dev, q->ndesc * sizeof(*q->desc),
++ &dma_addr, GFP_KERNEL);
++ if (!q->desc)
++ return -ENOMEM;
++
++ for (i = 0; i < q->ndesc; i++) {
++ u32 val;
++
++ val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1);
++ WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val));
++ }
++
++ airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr);
++ airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,
++ FIELD_PREP(TX_RING_CPU_IDX_MASK, q->head));
++ airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,
++ FIELD_PREP(TX_RING_DMA_IDX_MASK, q->head));
++
++ return 0;
++}
++
++static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q,
++ struct airoha_qdma *qdma, int size)
++{
++ int id = irq_q - &qdma->q_tx_irq[0];
++ struct airoha_eth *eth = qdma->eth;
++ dma_addr_t dma_addr;
++
++ netif_napi_add_tx(eth->napi_dev, &irq_q->napi,
++ airoha_qdma_tx_napi_poll);
++ irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32),
++ &dma_addr, GFP_KERNEL);
++ if (!irq_q->q)
++ return -ENOMEM;
++
++ memset(irq_q->q, 0xff, size * sizeof(u32));
++ irq_q->size = size;
++ irq_q->qdma = qdma;
++
++ airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr);
++ airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK,
++ FIELD_PREP(TX_IRQ_DEPTH_MASK, size));
++ airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK,
++ FIELD_PREP(TX_IRQ_THR_MASK, 1));
++
++ return 0;
++}
++
++static int airoha_qdma_init_tx(struct airoha_qdma *qdma)
++{
++ int i, err;
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
++ err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma,
++ IRQ_QUEUE_LEN(i));
++ if (err)
++ return err;
++ }
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
++ err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma,
++ TX_DSCP_NUM);
++ if (err)
++ return err;
++ }
++
++ return 0;
++}
++
++static void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q)
++{
++ struct airoha_eth *eth = q->qdma->eth;
++
++ spin_lock_bh(&q->lock);
++ while (q->queued) {
++ struct airoha_queue_entry *e = &q->entry[q->tail];
++
++ dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
++ DMA_TO_DEVICE);
++ dev_kfree_skb_any(e->skb);
++ e->skb = NULL;
++
++ q->tail = (q->tail + 1) % q->ndesc;
++ q->queued--;
++ }
++ spin_unlock_bh(&q->lock);
++}
++
++static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma)
++{
++ struct airoha_eth *eth = qdma->eth;
++ dma_addr_t dma_addr;
++ u32 status;
++ int size;
++
++ size = HW_DSCP_NUM * sizeof(struct airoha_qdma_fwd_desc);
++ qdma->hfwd.desc = dmam_alloc_coherent(eth->dev, size, &dma_addr,
++ GFP_KERNEL);
++ if (!qdma->hfwd.desc)
++ return -ENOMEM;
++
++ airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr);
++
++ size = AIROHA_MAX_PACKET_SIZE * HW_DSCP_NUM;
++ qdma->hfwd.q = dmam_alloc_coherent(eth->dev, size, &dma_addr,
++ GFP_KERNEL);
++ if (!qdma->hfwd.q)
++ return -ENOMEM;
++
++ airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr);
++
++ airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG,
++ HW_FWD_DSCP_PAYLOAD_SIZE_MASK,
++ FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK, 0));
++ airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK,
++ FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128));
++ airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG,
++ LMGR_INIT_START | LMGR_SRAM_MODE_MASK |
++ HW_FWD_DESC_NUM_MASK,
++ FIELD_PREP(HW_FWD_DESC_NUM_MASK, HW_DSCP_NUM) |
++ LMGR_INIT_START);
++
++ return read_poll_timeout(airoha_qdma_rr, status,
++ !(status & LMGR_INIT_START), USEC_PER_MSEC,
++ 30 * USEC_PER_MSEC, true, qdma,
++ REG_LMGR_INIT_CFG);
++}
++
++static void airoha_qdma_init_qos(struct airoha_qdma *qdma)
++{
++ airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK);
++ airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK);
++
++ airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG,
++ PSE_BUF_ESTIMATE_EN_MASK);
++
++ airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG,
++ EGRESS_RATE_METER_EN_MASK |
++ EGRESS_RATE_METER_EQ_RATE_EN_MASK);
++ /* 2047us x 31 = 63.457ms */
++ airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
++ EGRESS_RATE_METER_WINDOW_SZ_MASK,
++ FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK, 0x1f));
++ airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
++ EGRESS_RATE_METER_TIMESLICE_MASK,
++ FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK, 0x7ff));
++
++ /* ratelimit init */
++ airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK);
++ /* fast-tick 25us */
++ airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK,
++ FIELD_PREP(GLB_FAST_TICK_MASK, 25));
++ airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK,
++ FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40));
++
++ airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK);
++ airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK,
++ FIELD_PREP(EGRESS_FAST_TICK_MASK, 25));
++ airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG,
++ EGRESS_SLOW_TICK_RATIO_MASK,
++ FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40));
++
++ airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK);
++ airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG,
++ INGRESS_TRTCM_MODE_MASK);
++ airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK,
++ FIELD_PREP(INGRESS_FAST_TICK_MASK, 125));
++ airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG,
++ INGRESS_SLOW_TICK_RATIO_MASK,
++ FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8));
++
++ airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK);
++ airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK,
++ FIELD_PREP(SLA_FAST_TICK_MASK, 25));
++ airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK,
++ FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40));
++}
++
++static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma)
++{
++ int i;
++
++ for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) {
++ /* Tx-cpu transferred count */
++ airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0);
++ airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1),
++ CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
++ CNTR_ALL_DSCP_RING_EN_MASK |
++ FIELD_PREP(CNTR_CHAN_MASK, i));
++ /* Tx-fwd transferred count */
++ airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0);
++ airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1),
++ CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
++ CNTR_ALL_DSCP_RING_EN_MASK |
++ FIELD_PREP(CNTR_SRC_MASK, 1) |
++ FIELD_PREP(CNTR_CHAN_MASK, i));
++ }
++}
++
++static int airoha_qdma_hw_init(struct airoha_qdma *qdma)
++{
++ int i;
++
++ /* clear pending irqs */
++ for (i = 0; i < ARRAY_SIZE(qdma->irqmask); i++)
++ airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff);
++
++ /* setup irqs */
++ airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX0, INT_IDX0_MASK);
++ airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX1, INT_IDX1_MASK);
++ airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX4, INT_IDX4_MASK);
++
++ /* setup irq binding */
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
++ if (!qdma->q_tx[i].ndesc)
++ continue;
++
++ if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i))
++ airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i),
++ TX_RING_IRQ_BLOCKING_CFG_MASK);
++ else
++ airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i),
++ TX_RING_IRQ_BLOCKING_CFG_MASK);
++ }
++
++ airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG,
++ GLOBAL_CFG_RX_2B_OFFSET_MASK |
++ FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) |
++ GLOBAL_CFG_CPU_TXR_RR_MASK |
++ GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK |
++ GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK |
++ GLOBAL_CFG_MULTICAST_EN_MASK |
++ GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK |
++ GLOBAL_CFG_TX_WB_DONE_MASK |
++ FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2));
++
++ airoha_qdma_init_qos(qdma);
++
++ /* disable qdma rx delay interrupt */
++ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
++ if (!qdma->q_rx[i].ndesc)
++ continue;
++
++ airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i),
++ RX_DELAY_INT_MASK);
++ }
++
++ airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG,
++ TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN);
++ airoha_qdma_init_qos_stats(qdma);
++
++ return 0;
++}
++
++static irqreturn_t airoha_irq_handler(int irq, void *dev_instance)
++{
++ struct airoha_qdma *qdma = dev_instance;
++ u32 intr[ARRAY_SIZE(qdma->irqmask)];
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(qdma->irqmask); i++) {
++ intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i));
++ intr[i] &= qdma->irqmask[i];
++ airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]);
++ }
++
++ if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state))
++ return IRQ_NONE;
++
++ if (intr[1] & RX_DONE_INT_MASK) {
++ airoha_qdma_irq_disable(qdma, QDMA_INT_REG_IDX1,
++ RX_DONE_INT_MASK);
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
++ if (!qdma->q_rx[i].ndesc)
++ continue;
++
++ if (intr[1] & BIT(i))
++ napi_schedule(&qdma->q_rx[i].napi);
++ }
++ }
++
++ if (intr[0] & INT_TX_MASK) {
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
++ if (!(intr[0] & TX_DONE_INT_MASK(i)))
++ continue;
++
++ airoha_qdma_irq_disable(qdma, QDMA_INT_REG_IDX0,
++ TX_DONE_INT_MASK(i));
++ napi_schedule(&qdma->q_tx_irq[i].napi);
++ }
++ }
++
++ return IRQ_HANDLED;
++}
++
++static int airoha_qdma_init(struct platform_device *pdev,
++ struct airoha_eth *eth,
++ struct airoha_qdma *qdma)
++{
++ int err, id = qdma - ð->qdma[0];
++ const char *res;
++
++ spin_lock_init(&qdma->irq_lock);
++ qdma->eth = eth;
++
++ res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id);
++ if (!res)
++ return -ENOMEM;
++
++ qdma->regs = devm_platform_ioremap_resource_byname(pdev, res);
++ if (IS_ERR(qdma->regs))
++ return dev_err_probe(eth->dev, PTR_ERR(qdma->regs),
++ "failed to iomap qdma%d regs\n", id);
++
++ qdma->irq = platform_get_irq(pdev, 4 * id);
++ if (qdma->irq < 0)
++ return qdma->irq;
++
++ err = devm_request_irq(eth->dev, qdma->irq, airoha_irq_handler,
++ IRQF_SHARED, KBUILD_MODNAME, qdma);
++ if (err)
++ return err;
++
++ err = airoha_qdma_init_rx(qdma);
++ if (err)
++ return err;
++
++ err = airoha_qdma_init_tx(qdma);
++ if (err)
++ return err;
++
++ err = airoha_qdma_init_hfwd_queues(qdma);
++ if (err)
++ return err;
++
++ return airoha_qdma_hw_init(qdma);
++}
++
++static int airoha_hw_init(struct platform_device *pdev,
++ struct airoha_eth *eth)
++{
++ int err, i;
++
++ /* disable xsi */
++ err = reset_control_bulk_assert(ARRAY_SIZE(eth->xsi_rsts),
++ eth->xsi_rsts);
++ if (err)
++ return err;
++
++ err = reset_control_bulk_assert(ARRAY_SIZE(eth->rsts), eth->rsts);
++ if (err)
++ return err;
++
++ msleep(20);
++ err = reset_control_bulk_deassert(ARRAY_SIZE(eth->rsts), eth->rsts);
++ if (err)
++ return err;
++
++ msleep(20);
++ err = airoha_fe_init(eth);
++ if (err)
++ return err;
++
++ for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
++ err = airoha_qdma_init(pdev, eth, ð->qdma[i]);
++ if (err)
++ return err;
++ }
++
++ set_bit(DEV_STATE_INITIALIZED, ð->state);
++
++ return 0;
++}
++
++static void airoha_hw_cleanup(struct airoha_qdma *qdma)
++{
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
++ if (!qdma->q_rx[i].ndesc)
++ continue;
++
++ netif_napi_del(&qdma->q_rx[i].napi);
++ airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]);
++ if (qdma->q_rx[i].page_pool)
++ page_pool_destroy(qdma->q_rx[i].page_pool);
++ }
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
++ netif_napi_del(&qdma->q_tx_irq[i].napi);
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
++ if (!qdma->q_tx[i].ndesc)
++ continue;
++
++ airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);
++ }
++}
++
++static void airoha_qdma_start_napi(struct airoha_qdma *qdma)
++{
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
++ napi_enable(&qdma->q_tx_irq[i].napi);
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
++ if (!qdma->q_rx[i].ndesc)
++ continue;
++
++ napi_enable(&qdma->q_rx[i].napi);
++ }
++}
++
++static void airoha_qdma_stop_napi(struct airoha_qdma *qdma)
++{
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
++ napi_disable(&qdma->q_tx_irq[i].napi);
++
++ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
++ if (!qdma->q_rx[i].ndesc)
++ continue;
++
++ napi_disable(&qdma->q_rx[i].napi);
++ }
++}
++
++static void airoha_update_hw_stats(struct airoha_gdm_port *port)
++{
++ struct airoha_eth *eth = port->qdma->eth;
++ u32 val, i = 0;
++
++ spin_lock(&port->stats.lock);
++ u64_stats_update_begin(&port->stats.syncp);
++
++ /* TX */
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_H(port->id));
++ port->stats.tx_ok_pkts += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_L(port->id));
++ port->stats.tx_ok_pkts += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_H(port->id));
++ port->stats.tx_ok_bytes += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_L(port->id));
++ port->stats.tx_ok_bytes += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_DROP_CNT(port->id));
++ port->stats.tx_drops += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_BC_CNT(port->id));
++ port->stats.tx_broadcast += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_MC_CNT(port->id));
++ port->stats.tx_multicast += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_RUNT_CNT(port->id));
++ port->stats.tx_len[i] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_H(port->id));
++ port->stats.tx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_L(port->id));
++ port->stats.tx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_H(port->id));
++ port->stats.tx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_L(port->id));
++ port->stats.tx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_H(port->id));
++ port->stats.tx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_L(port->id));
++ port->stats.tx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_H(port->id));
++ port->stats.tx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_L(port->id));
++ port->stats.tx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_H(port->id));
++ port->stats.tx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_L(port->id));
++ port->stats.tx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_H(port->id));
++ port->stats.tx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_L(port->id));
++ port->stats.tx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_LONG_CNT(port->id));
++ port->stats.tx_len[i++] += val;
++
++ /* RX */
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_H(port->id));
++ port->stats.rx_ok_pkts += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_L(port->id));
++ port->stats.rx_ok_pkts += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_H(port->id));
++ port->stats.rx_ok_bytes += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_L(port->id));
++ port->stats.rx_ok_bytes += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_DROP_CNT(port->id));
++ port->stats.rx_drops += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_BC_CNT(port->id));
++ port->stats.rx_broadcast += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_MC_CNT(port->id));
++ port->stats.rx_multicast += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ERROR_DROP_CNT(port->id));
++ port->stats.rx_errors += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_CRC_ERR_CNT(port->id));
++ port->stats.rx_crc_error += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_OVERFLOW_DROP_CNT(port->id));
++ port->stats.rx_over_errors += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_FRAG_CNT(port->id));
++ port->stats.rx_fragment += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_JABBER_CNT(port->id));
++ port->stats.rx_jabber += val;
++
++ i = 0;
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_RUNT_CNT(port->id));
++ port->stats.rx_len[i] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_H(port->id));
++ port->stats.rx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_L(port->id));
++ port->stats.rx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_H(port->id));
++ port->stats.rx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_L(port->id));
++ port->stats.rx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_H(port->id));
++ port->stats.rx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_L(port->id));
++ port->stats.rx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_H(port->id));
++ port->stats.rx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_L(port->id));
++ port->stats.rx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_H(port->id));
++ port->stats.rx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_L(port->id));
++ port->stats.rx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_H(port->id));
++ port->stats.rx_len[i] += ((u64)val << 32);
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_L(port->id));
++ port->stats.rx_len[i++] += val;
++
++ val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_LONG_CNT(port->id));
++ port->stats.rx_len[i++] += val;
++
++ /* reset mib counters */
++ airoha_fe_set(eth, REG_FE_GDM_MIB_CLEAR(port->id),
++ FE_GDM_MIB_RX_CLEAR_MASK | FE_GDM_MIB_TX_CLEAR_MASK);
++
++ u64_stats_update_end(&port->stats.syncp);
++ spin_unlock(&port->stats.lock);
++}
++
++static int airoha_dev_open(struct net_device *dev)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ struct airoha_qdma *qdma = port->qdma;
++ int err;
++
++ netif_tx_start_all_queues(dev);
++ err = airoha_set_gdm_ports(qdma->eth, true);
++ if (err)
++ return err;
++
++ if (netdev_uses_dsa(dev))
++ airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id),
++ GDM_STAG_EN_MASK);
++ else
++ airoha_fe_clear(qdma->eth, REG_GDM_INGRESS_CFG(port->id),
++ GDM_STAG_EN_MASK);
++
++ airoha_qdma_set(qdma, REG_QDMA_GLOBAL_CFG,
++ GLOBAL_CFG_TX_DMA_EN_MASK |
++ GLOBAL_CFG_RX_DMA_EN_MASK);
++
++ return 0;
++}
++
++static int airoha_dev_stop(struct net_device *dev)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ struct airoha_qdma *qdma = port->qdma;
++ int err;
++
++ netif_tx_disable(dev);
++ err = airoha_set_gdm_ports(qdma->eth, false);
++ if (err)
++ return err;
++
++ airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,
++ GLOBAL_CFG_TX_DMA_EN_MASK |
++ GLOBAL_CFG_RX_DMA_EN_MASK);
++
++ return 0;
++}
++
++static int airoha_dev_set_macaddr(struct net_device *dev, void *p)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ int err;
++
++ err = eth_mac_addr(dev, p);
++ if (err)
++ return err;
++
++ airoha_set_macaddr(port, dev->dev_addr);
++
++ return 0;
++}
++
++static int airoha_dev_init(struct net_device *dev)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++
++ airoha_set_macaddr(port, dev->dev_addr);
++
++ return 0;
++}
++
++static void airoha_dev_get_stats64(struct net_device *dev,
++ struct rtnl_link_stats64 *storage)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ unsigned int start;
++
++ airoha_update_hw_stats(port);
++ do {
++ start = u64_stats_fetch_begin(&port->stats.syncp);
++ storage->rx_packets = port->stats.rx_ok_pkts;
++ storage->tx_packets = port->stats.tx_ok_pkts;
++ storage->rx_bytes = port->stats.rx_ok_bytes;
++ storage->tx_bytes = port->stats.tx_ok_bytes;
++ storage->multicast = port->stats.rx_multicast;
++ storage->rx_errors = port->stats.rx_errors;
++ storage->rx_dropped = port->stats.rx_drops;
++ storage->tx_dropped = port->stats.tx_drops;
++ storage->rx_crc_errors = port->stats.rx_crc_error;
++ storage->rx_over_errors = port->stats.rx_over_errors;
++ } while (u64_stats_fetch_retry(&port->stats.syncp, start));
++}
++
++static u16 airoha_dev_select_queue(struct net_device *dev, struct sk_buff *skb,
++ struct net_device *sb_dev)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ int queue, channel;
++
++ /* For dsa device select QoS channel according to the dsa user port
++ * index, rely on port id otherwise. Select QoS queue based on the
++ * skb priority.
++ */
++ channel = netdev_uses_dsa(dev) ? skb_get_queue_mapping(skb) : port->id;
++ channel = channel % AIROHA_NUM_QOS_CHANNELS;
++ queue = skb->priority % AIROHA_NUM_QOS_QUEUES;
++ queue = channel * AIROHA_NUM_QOS_QUEUES + queue;
++
++ return queue < dev->num_tx_queues ? queue : 0;
++}
++
++static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
++ struct net_device *dev)
++{
++ struct skb_shared_info *sinfo = skb_shinfo(skb);
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ u32 msg0, msg1, len = skb_headlen(skb);
++ struct airoha_qdma *qdma = port->qdma;
++ u32 nr_frags = 1 + sinfo->nr_frags;
++ struct netdev_queue *txq;
++ struct airoha_queue *q;
++ void *data = skb->data;
++ int i, qid;
++ u16 index;
++ u8 fport;
++
++ qid = skb_get_queue_mapping(skb) % ARRAY_SIZE(qdma->q_tx);
++ msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
++ qid / AIROHA_NUM_QOS_QUEUES) |
++ FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
++ qid % AIROHA_NUM_QOS_QUEUES);
++ if (skb->ip_summed == CHECKSUM_PARTIAL)
++ msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TCO_MASK, 1) |
++ FIELD_PREP(QDMA_ETH_TXMSG_UCO_MASK, 1) |
++ FIELD_PREP(QDMA_ETH_TXMSG_ICO_MASK, 1);
++
++ /* TSO: fill MSS info in tcp checksum field */
++ if (skb_is_gso(skb)) {
++ if (skb_cow_head(skb, 0))
++ goto error;
++
++ if (sinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) {
++ __be16 csum = cpu_to_be16(sinfo->gso_size);
++
++ tcp_hdr(skb)->check = (__force __sum16)csum;
++ msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TSO_MASK, 1);
++ }
++ }
++
++ fport = port->id == 4 ? FE_PSE_PORT_GDM4 : port->id;
++ msg1 = FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
++ FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
++
++ q = &qdma->q_tx[qid];
++ if (WARN_ON_ONCE(!q->ndesc))
++ goto error;
++
++ spin_lock_bh(&q->lock);
++
++ txq = netdev_get_tx_queue(dev, qid);
++ if (q->queued + nr_frags > q->ndesc) {
++ /* not enough space in the queue */
++ netif_tx_stop_queue(txq);
++ spin_unlock_bh(&q->lock);
++ return NETDEV_TX_BUSY;
++ }
++
++ index = q->head;
++ for (i = 0; i < nr_frags; i++) {
++ struct airoha_qdma_desc *desc = &q->desc[index];
++ struct airoha_queue_entry *e = &q->entry[index];
++ skb_frag_t *frag = &sinfo->frags[i];
++ dma_addr_t addr;
++ u32 val;
++
++ addr = dma_map_single(dev->dev.parent, data, len,
++ DMA_TO_DEVICE);
++ if (unlikely(dma_mapping_error(dev->dev.parent, addr)))
++ goto error_unmap;
++
++ index = (index + 1) % q->ndesc;
++
++ val = FIELD_PREP(QDMA_DESC_LEN_MASK, len);
++ if (i < nr_frags - 1)
++ val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1);
++ WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
++ WRITE_ONCE(desc->addr, cpu_to_le32(addr));
++ val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, index);
++ WRITE_ONCE(desc->data, cpu_to_le32(val));
++ WRITE_ONCE(desc->msg0, cpu_to_le32(msg0));
++ WRITE_ONCE(desc->msg1, cpu_to_le32(msg1));
++ WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
++
++ e->skb = i ? NULL : skb;
++ e->dma_addr = addr;
++ e->dma_len = len;
++
++ data = skb_frag_address(frag);
++ len = skb_frag_size(frag);
++ }
++
++ q->head = index;
++ q->queued += i;
++
++ skb_tx_timestamp(skb);
++ netdev_tx_sent_queue(txq, skb->len);
++
++ if (netif_xmit_stopped(txq) || !netdev_xmit_more())
++ airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
++ TX_RING_CPU_IDX_MASK,
++ FIELD_PREP(TX_RING_CPU_IDX_MASK, q->head));
++
++ if (q->ndesc - q->queued < q->free_thr)
++ netif_tx_stop_queue(txq);
++
++ spin_unlock_bh(&q->lock);
++
++ return NETDEV_TX_OK;
++
++error_unmap:
++ for (i--; i >= 0; i--) {
++ index = (q->head + i) % q->ndesc;
++ dma_unmap_single(dev->dev.parent, q->entry[index].dma_addr,
++ q->entry[index].dma_len, DMA_TO_DEVICE);
++ }
++
++ spin_unlock_bh(&q->lock);
++error:
++ dev_kfree_skb_any(skb);
++ dev->stats.tx_dropped++;
++
++ return NETDEV_TX_OK;
++}
++
++static void airoha_ethtool_get_drvinfo(struct net_device *dev,
++ struct ethtool_drvinfo *info)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ struct airoha_eth *eth = port->qdma->eth;
++
++ strscpy(info->driver, eth->dev->driver->name, sizeof(info->driver));
++ strscpy(info->bus_info, dev_name(eth->dev), sizeof(info->bus_info));
++}
++
++static void airoha_ethtool_get_mac_stats(struct net_device *dev,
++ struct ethtool_eth_mac_stats *stats)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ unsigned int start;
++
++ airoha_update_hw_stats(port);
++ do {
++ start = u64_stats_fetch_begin(&port->stats.syncp);
++ stats->MulticastFramesXmittedOK = port->stats.tx_multicast;
++ stats->BroadcastFramesXmittedOK = port->stats.tx_broadcast;
++ stats->BroadcastFramesReceivedOK = port->stats.rx_broadcast;
++ } while (u64_stats_fetch_retry(&port->stats.syncp, start));
++}
++
++static const struct ethtool_rmon_hist_range airoha_ethtool_rmon_ranges[] = {
++ { 0, 64 },
++ { 65, 127 },
++ { 128, 255 },
++ { 256, 511 },
++ { 512, 1023 },
++ { 1024, 1518 },
++ { 1519, 10239 },
++ {},
++};
++
++static void
++airoha_ethtool_get_rmon_stats(struct net_device *dev,
++ struct ethtool_rmon_stats *stats,
++ const struct ethtool_rmon_hist_range **ranges)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++ struct airoha_hw_stats *hw_stats = &port->stats;
++ unsigned int start;
++
++ BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) !=
++ ARRAY_SIZE(hw_stats->tx_len) + 1);
++ BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) !=
++ ARRAY_SIZE(hw_stats->rx_len) + 1);
++
++ *ranges = airoha_ethtool_rmon_ranges;
++ airoha_update_hw_stats(port);
++ do {
++ int i;
++
++ start = u64_stats_fetch_begin(&port->stats.syncp);
++ stats->fragments = hw_stats->rx_fragment;
++ stats->jabbers = hw_stats->rx_jabber;
++ for (i = 0; i < ARRAY_SIZE(airoha_ethtool_rmon_ranges) - 1;
++ i++) {
++ stats->hist[i] = hw_stats->rx_len[i];
++ stats->hist_tx[i] = hw_stats->tx_len[i];
++ }
++ } while (u64_stats_fetch_retry(&port->stats.syncp, start));
++}
++
++static int airoha_qdma_set_chan_tx_sched(struct airoha_gdm_port *port,
++ int channel, enum tx_sched_mode mode,
++ const u16 *weights, u8 n_weights)
++{
++ int i;
++
++ for (i = 0; i < AIROHA_NUM_TX_RING; i++)
++ airoha_qdma_clear(port->qdma, REG_QUEUE_CLOSE_CFG(channel),
++ TXQ_DISABLE_CHAN_QUEUE_MASK(channel, i));
++
++ for (i = 0; i < n_weights; i++) {
++ u32 status;
++ int err;
++
++ airoha_qdma_wr(port->qdma, REG_TXWRR_WEIGHT_CFG,
++ TWRR_RW_CMD_MASK |
++ FIELD_PREP(TWRR_CHAN_IDX_MASK, channel) |
++ FIELD_PREP(TWRR_QUEUE_IDX_MASK, i) |
++ FIELD_PREP(TWRR_VALUE_MASK, weights[i]));
++ err = read_poll_timeout(airoha_qdma_rr, status,
++ status & TWRR_RW_CMD_DONE,
++ USEC_PER_MSEC, 10 * USEC_PER_MSEC,
++ true, port->qdma,
++ REG_TXWRR_WEIGHT_CFG);
++ if (err)
++ return err;
++ }
++
++ airoha_qdma_rmw(port->qdma, REG_CHAN_QOS_MODE(channel >> 3),
++ CHAN_QOS_MODE_MASK(channel),
++ mode << __ffs(CHAN_QOS_MODE_MASK(channel)));
++
++ return 0;
++}
++
++static int airoha_qdma_set_tx_prio_sched(struct airoha_gdm_port *port,
++ int channel)
++{
++ static const u16 w[AIROHA_NUM_QOS_QUEUES] = {};
++
++ return airoha_qdma_set_chan_tx_sched(port, channel, TC_SCH_SP, w,
++ ARRAY_SIZE(w));
++}
++
++static int airoha_qdma_set_tx_ets_sched(struct airoha_gdm_port *port,
++ int channel,
++ struct tc_ets_qopt_offload *opt)
++{
++ struct tc_ets_qopt_offload_replace_params *p = &opt->replace_params;
++ enum tx_sched_mode mode = TC_SCH_SP;
++ u16 w[AIROHA_NUM_QOS_QUEUES] = {};
++ int i, nstrict = 0;
++
++ if (p->bands > AIROHA_NUM_QOS_QUEUES)
++ return -EINVAL;
++
++ for (i = 0; i < p->bands; i++) {
++ if (!p->quanta[i])
++ nstrict++;
++ }
++
++ /* this configuration is not supported by the hw */
++ if (nstrict == AIROHA_NUM_QOS_QUEUES - 1)
++ return -EINVAL;
++
++ for (i = 0; i < p->bands - nstrict; i++)
++ w[i] = p->weights[nstrict + i];
++
++ if (!nstrict)
++ mode = TC_SCH_WRR8;
++ else if (nstrict < AIROHA_NUM_QOS_QUEUES - 1)
++ mode = nstrict + 1;
++
++ return airoha_qdma_set_chan_tx_sched(port, channel, mode, w,
++ ARRAY_SIZE(w));
++}
++
++static int airoha_qdma_get_tx_ets_stats(struct airoha_gdm_port *port,
++ int channel,
++ struct tc_ets_qopt_offload *opt)
++{
++ u64 cpu_tx_packets = airoha_qdma_rr(port->qdma,
++ REG_CNTR_VAL(channel << 1));
++ u64 fwd_tx_packets = airoha_qdma_rr(port->qdma,
++ REG_CNTR_VAL((channel << 1) + 1));
++ u64 tx_packets = (cpu_tx_packets - port->cpu_tx_packets) +
++ (fwd_tx_packets - port->fwd_tx_packets);
++ _bstats_update(opt->stats.bstats, 0, tx_packets);
++
++ port->cpu_tx_packets = cpu_tx_packets;
++ port->fwd_tx_packets = fwd_tx_packets;
++
++ return 0;
++}
++
++static int airoha_tc_setup_qdisc_ets(struct airoha_gdm_port *port,
++ struct tc_ets_qopt_offload *opt)
++{
++ int channel;
++
++ if (opt->parent == TC_H_ROOT)
++ return -EINVAL;
++
++ channel = TC_H_MIN(opt->parent) % AIROHA_NUM_QOS_CHANNELS;
++
++ switch (opt->command) {
++ case TC_ETS_REPLACE:
++ return airoha_qdma_set_tx_ets_sched(port, channel, opt);
++ case TC_ETS_DESTROY:
++ /* PRIO is default qdisc scheduler */
++ return airoha_qdma_set_tx_prio_sched(port, channel);
++ case TC_ETS_STATS:
++ return airoha_qdma_get_tx_ets_stats(port, channel, opt);
++ default:
++ return -EOPNOTSUPP;
++ }
++}
++
++static int airoha_dev_tc_setup(struct net_device *dev, enum tc_setup_type type,
++ void *type_data)
++{
++ struct airoha_gdm_port *port = netdev_priv(dev);
++
++ switch (type) {
++ case TC_SETUP_QDISC_ETS:
++ return airoha_tc_setup_qdisc_ets(port, type_data);
++ default:
++ return -EOPNOTSUPP;
++ }
++}
++
++static const struct net_device_ops airoha_netdev_ops = {
++ .ndo_init = airoha_dev_init,
++ .ndo_open = airoha_dev_open,
++ .ndo_stop = airoha_dev_stop,
++ .ndo_select_queue = airoha_dev_select_queue,
++ .ndo_start_xmit = airoha_dev_xmit,
++ .ndo_get_stats64 = airoha_dev_get_stats64,
++ .ndo_set_mac_address = airoha_dev_set_macaddr,
++ .ndo_setup_tc = airoha_dev_tc_setup,
++};
++
++static const struct ethtool_ops airoha_ethtool_ops = {
++ .get_drvinfo = airoha_ethtool_get_drvinfo,
++ .get_eth_mac_stats = airoha_ethtool_get_mac_stats,
++ .get_rmon_stats = airoha_ethtool_get_rmon_stats,
++};
++
++static int airoha_alloc_gdm_port(struct airoha_eth *eth, struct device_node *np)
++{
++ const __be32 *id_ptr = of_get_property(np, "reg", NULL);
++ struct airoha_gdm_port *port;
++ struct airoha_qdma *qdma;
++ struct net_device *dev;
++ int err, index;
++ u32 id;
++
++ if (!id_ptr) {
++ dev_err(eth->dev, "missing gdm port id\n");
++ return -EINVAL;
++ }
++
++ id = be32_to_cpup(id_ptr);
++ index = id - 1;
++
++ if (!id || id > ARRAY_SIZE(eth->ports)) {
++ dev_err(eth->dev, "invalid gdm port id: %d\n", id);
++ return -EINVAL;
++ }
++
++ if (eth->ports[index]) {
++ dev_err(eth->dev, "duplicate gdm port id: %d\n", id);
++ return -EINVAL;
++ }
++
++ dev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*port),
++ AIROHA_NUM_TX_RING, AIROHA_NUM_RX_RING);
++ if (!dev) {
++ dev_err(eth->dev, "alloc_etherdev failed\n");
++ return -ENOMEM;
++ }
++
++ qdma = ð->qdma[index % AIROHA_MAX_NUM_QDMA];
++ dev->netdev_ops = &airoha_netdev_ops;
++ dev->ethtool_ops = &airoha_ethtool_ops;
++ dev->max_mtu = AIROHA_MAX_MTU;
++ dev->watchdog_timeo = 5 * HZ;
++ dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
++ NETIF_F_TSO6 | NETIF_F_IPV6_CSUM |
++ NETIF_F_SG | NETIF_F_TSO |
++ NETIF_F_HW_TC;
++ dev->features |= dev->hw_features;
++ dev->dev.of_node = np;
++ dev->irq = qdma->irq;
++ SET_NETDEV_DEV(dev, eth->dev);
++
++ err = of_get_ethdev_address(np, dev);
++ if (err) {
++ if (err == -EPROBE_DEFER)
++ return err;
++
++ eth_hw_addr_random(dev);
++ dev_info(eth->dev, "generated random MAC address %pM\n",
++ dev->dev_addr);
++ }
++
++ port = netdev_priv(dev);
++ u64_stats_init(&port->stats.syncp);
++ spin_lock_init(&port->stats.lock);
++ port->qdma = qdma;
++ port->dev = dev;
++ port->id = id;
++ eth->ports[index] = port;
++
++ return register_netdev(dev);
++}
++
++static int airoha_probe(struct platform_device *pdev)
++{
++ struct device_node *np;
++ struct airoha_eth *eth;
++ int i, err;
++
++ eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL);
++ if (!eth)
++ return -ENOMEM;
++
++ eth->dev = &pdev->dev;
++
++ err = dma_set_mask_and_coherent(eth->dev, DMA_BIT_MASK(32));
++ if (err) {
++ dev_err(eth->dev, "failed configuring DMA mask\n");
++ return err;
++ }
++
++ eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe");
++ if (IS_ERR(eth->fe_regs))
++ return dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs),
++ "failed to iomap fe regs\n");
++
++ eth->rsts[0].id = "fe";
++ eth->rsts[1].id = "pdma";
++ eth->rsts[2].id = "qdma";
++ err = devm_reset_control_bulk_get_exclusive(eth->dev,
++ ARRAY_SIZE(eth->rsts),
++ eth->rsts);
++ if (err) {
++ dev_err(eth->dev, "failed to get bulk reset lines\n");
++ return err;
++ }
++
++ eth->xsi_rsts[0].id = "xsi-mac";
++ eth->xsi_rsts[1].id = "hsi0-mac";
++ eth->xsi_rsts[2].id = "hsi1-mac";
++ eth->xsi_rsts[3].id = "hsi-mac";
++ eth->xsi_rsts[4].id = "xfp-mac";
++ err = devm_reset_control_bulk_get_exclusive(eth->dev,
++ ARRAY_SIZE(eth->xsi_rsts),
++ eth->xsi_rsts);
++ if (err) {
++ dev_err(eth->dev, "failed to get bulk xsi reset lines\n");
++ return err;
++ }
++
++ eth->napi_dev = alloc_netdev_dummy(0);
++ if (!eth->napi_dev)
++ return -ENOMEM;
++
++ /* Enable threaded NAPI by default */
++ eth->napi_dev->threaded = true;
++ strscpy(eth->napi_dev->name, "qdma_eth", sizeof(eth->napi_dev->name));
++ platform_set_drvdata(pdev, eth);
++
++ err = airoha_hw_init(pdev, eth);
++ if (err)
++ goto error_hw_cleanup;
++
++ for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
++ airoha_qdma_start_napi(ð->qdma[i]);
++
++ for_each_child_of_node(pdev->dev.of_node, np) {
++ if (!of_device_is_compatible(np, "airoha,eth-mac"))
++ continue;
++
++ if (!of_device_is_available(np))
++ continue;
++
++ err = airoha_alloc_gdm_port(eth, np);
++ if (err) {
++ of_node_put(np);
++ goto error_napi_stop;
++ }
++ }
++
++ return 0;
++
++error_napi_stop:
++ for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
++ airoha_qdma_stop_napi(ð->qdma[i]);
++error_hw_cleanup:
++ for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
++ airoha_hw_cleanup(ð->qdma[i]);
++
++ for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
++ struct airoha_gdm_port *port = eth->ports[i];
++
++ if (port && port->dev->reg_state == NETREG_REGISTERED)
++ unregister_netdev(port->dev);
++ }
++ free_netdev(eth->napi_dev);
++ platform_set_drvdata(pdev, NULL);
++
++ return err;
++}
++
++static void airoha_remove(struct platform_device *pdev)
++{
++ struct airoha_eth *eth = platform_get_drvdata(pdev);
++ int i;
++
++ for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
++ airoha_qdma_stop_napi(ð->qdma[i]);
++ airoha_hw_cleanup(ð->qdma[i]);
++ }
++
++ for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
++ struct airoha_gdm_port *port = eth->ports[i];
++
++ if (!port)
++ continue;
++
++ unregister_netdev(port->dev);
++ }
++ free_netdev(eth->napi_dev);
++
++ platform_set_drvdata(pdev, NULL);
++}
++
++static const struct of_device_id of_airoha_match[] = {
++ { .compatible = "airoha,en7581-eth" },
++ { /* sentinel */ }
++};
++MODULE_DEVICE_TABLE(of, of_airoha_match);
++
++static struct platform_driver airoha_driver = {
++ .probe = airoha_probe,
++ .remove_new = airoha_remove,
++ .driver = {
++ .name = KBUILD_MODNAME,
++ .of_match_table = of_airoha_match,
++ },
++};
++module_platform_driver(airoha_driver);
++
++MODULE_LICENSE("GPL");
++MODULE_AUTHOR("Lorenzo Bianconi <[email protected]>");
++MODULE_DESCRIPTION("Ethernet driver for Airoha SoC");
+diff --git a/drivers/net/ethernet/amd/pds_core/adminq.c b/drivers/net/ethernet/amd/pds_core/adminq.c
+index 506f682d15c10a..12eca570cb1da1 100644
+--- a/drivers/net/ethernet/amd/pds_core/adminq.c
++++ b/drivers/net/ethernet/amd/pds_core/adminq.c
+@@ -18,7 +18,13 @@ static int pdsc_process_notifyq(struct pdsc_qcq *qcq)
+ comp = cq_info->comp;
+ eid = le64_to_cpu(comp->event.eid);
+ while (eid > pdsc->last_eid) {
+- u16 ecode = le16_to_cpu(comp->event.ecode);
++ u16 ecode;
++
++ /* Order the payload read after the event id, the field the
++ * driver uses to detect a new completion.
++ */
++ dma_rmb();
++ ecode = le16_to_cpu(comp->event.ecode);
+
+ switch (ecode) {
+ case PDS_EVENT_LINK_CHANGE:
+@@ -101,6 +107,10 @@ void pdsc_process_adminq(struct pdsc_qcq *qcq)
+ spin_lock_irqsave(&pdsc->adminq_lock, irqflags);
+ comp = cq->info[cq->tail_idx].comp;
+ while (pdsc_color_match(comp->color, cq->done_color)) {
++ /* Order the payload reads after the color bit, the field the
++ * driver uses to detect a new completion.
++ */
++ dma_rmb();
+ q_info = &q->info[q->tail_idx];
+ q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
+
+diff --git a/drivers/net/ethernet/amd/pds_core/auxbus.c b/drivers/net/ethernet/amd/pds_core/auxbus.c
+index 889a18962270aa..7d7c87fe9e8f27 100644
+--- a/drivers/net/ethernet/amd/pds_core/auxbus.c
++++ b/drivers/net/ethernet/amd/pds_core/auxbus.c
+@@ -177,17 +177,21 @@ void pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf,
+ {
+ struct pds_auxiliary_dev *padev;
+
+- if (!*pd_ptr)
+- return;
+-
+ mutex_lock(&pf->config_lock);
+
++ /* A concurrent del may have already torn this device down and
++ * cleared it.
++ */
+ padev = *pd_ptr;
++ if (!padev)
++ goto out_unlock;
++
+ pds_client_unregister(pf, padev->client_id);
+ auxiliary_device_delete(&padev->aux_dev);
+ auxiliary_device_uninit(&padev->aux_dev);
+ *pd_ptr = NULL;
+
++out_unlock:
+ mutex_unlock(&pf->config_lock);
+ }
+
+@@ -210,6 +214,13 @@ int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf,
+
+ mutex_lock(&pf->config_lock);
+
++ /* Nothing to do if the aux device is already present. This also
++ * guards against a second add overwriting *pd_ptr and leaking the
++ * first, symmetric with the check in pdsc_auxbus_dev_del().
++ */
++ if (*pd_ptr)
++ goto out_unlock;
++
+ mask = BIT_ULL(PDSC_S_FW_DEAD) |
+ BIT_ULL(PDSC_S_STOPPING_DRIVER);
+ if (cf->state & mask) {
+diff --git a/drivers/net/ethernet/amd/pds_core/core.c b/drivers/net/ethernet/amd/pds_core/core.c
+index 3c60d4cf9d0e17..276e61f043c5ec 100644
+--- a/drivers/net/ethernet/amd/pds_core/core.c
++++ b/drivers/net/ethernet/amd/pds_core/core.c
+@@ -110,7 +110,6 @@ static void pdsc_qcq_intr_free(struct pdsc *pdsc, struct pdsc_qcq *qcq)
+ return;
+
+ pdsc_intr_free(pdsc, qcq->intx);
+- qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED;
+ }
+
+ static int pdsc_qcq_intr_alloc(struct pdsc *pdsc, struct pdsc_qcq *qcq)
+@@ -145,6 +144,12 @@ void pdsc_qcq_free(struct pdsc *pdsc, struct pdsc_qcq *qcq)
+
+ pdsc_qcq_intr_free(pdsc, qcq);
+
++ /* Drain any work queued by ISR before it was freed above */
++ if (qcq->work.func)
++ cancel_work_sync(&qcq->work);
++
++ qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED;
++
+ if (qcq->q_base)
+ dma_free_coherent(dev, qcq->q_size,
+ qcq->q_base, qcq->q_base_pa);
+@@ -304,8 +309,11 @@ err_out:
+
+ static void pdsc_core_uninit(struct pdsc *pdsc)
+ {
+- pdsc_qcq_free(pdsc, &pdsc->notifyqcq);
++ /* Free adminqcq first: its work accesses notifyqcq, so we must
++ * disable its IRQ and drain its work before freeing notifyqcq.
++ */
+ pdsc_qcq_free(pdsc, &pdsc->adminqcq);
++ pdsc_qcq_free(pdsc, &pdsc->notifyqcq);
+
+ if (pdsc->kern_dbpage) {
+ iounmap(pdsc->kern_dbpage);
+@@ -472,8 +480,6 @@ void pdsc_teardown(struct pdsc *pdsc, bool removing)
+ {
+ if (!pdsc->pdev->is_virtfn)
+ pdsc_devcmd_reset(pdsc);
+- if (pdsc->adminqcq.work.func)
+- cancel_work_sync(&pdsc->adminqcq.work);
+
+ pdsc_core_uninit(pdsc);
+
+@@ -524,6 +530,7 @@ static void pdsc_adminq_wait_and_dec_once_unused(struct pdsc *pdsc)
+ dev_dbg_ratelimited(pdsc->dev, "%s: adminq in use\n",
+ __func__);
+ cpu_relax();
++ cond_resched();
+ }
+ }
+
+@@ -597,9 +604,10 @@ void pdsc_pci_reset_thread(struct work_struct *work)
+ struct pdsc *pdsc = container_of(work, struct pdsc, pci_reset_work);
+ struct pci_dev *pdev = pdsc->pdev;
+
+- pci_dev_get(pdev);
+- pci_reset_function(pdev);
+- pci_dev_put(pdev);
++ /* Use try variant to avoid deadlock with pdsc_remove().
++ * If lock is contended, the watchdog timer will retry.
++ */
++ pci_try_reset_function(pdev);
+ }
+
+ static void pdsc_check_pci_health(struct pdsc *pdsc)
+diff --git a/drivers/net/ethernet/amd/pds_core/devlink.c b/drivers/net/ethernet/amd/pds_core/devlink.c
+index 621791a3c543be..e35572f099c884 100644
+--- a/drivers/net/ethernet/amd/pds_core/devlink.c
++++ b/drivers/net/ethernet/amd/pds_core/devlink.c
+@@ -89,6 +89,12 @@ int pdsc_dl_flash_update(struct devlink *dl,
+ {
+ struct pdsc *pdsc = devlink_priv(dl);
+
++ if (params->component) {
++ NL_SET_ERR_MSG_MOD(extack,
++ "Component update not supported by this device");
++ return -EOPNOTSUPP;
++ }
++
+ return pdsc_firmware_update(pdsc, params->fw, extack);
+ }
+
+diff --git a/drivers/net/ethernet/amd/pds_core/main.c b/drivers/net/ethernet/amd/pds_core/main.c
+index a3a68889137b63..c78b17d230a841 100644
+--- a/drivers/net/ethernet/amd/pds_core/main.c
++++ b/drivers/net/ethernet/amd/pds_core/main.c
+@@ -239,6 +239,10 @@ static int pdsc_init_pf(struct pdsc *pdsc)
+ /* General workqueue and timer, but don't start timer yet */
+ snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
+ pdsc->wq = create_singlethread_workqueue(wq_name);
++ if (!pdsc->wq) {
++ err = -ENOMEM;
++ goto err_out_unmap_bars;
++ }
+ INIT_WORK(&pdsc->health_work, pdsc_health_thread);
+ INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);
+ timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
+@@ -254,7 +258,7 @@ static int pdsc_init_pf(struct pdsc *pdsc)
+ err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
+ if (err) {
+ mutex_unlock(&pdsc->config_lock);
+- goto err_out_unmap_bars;
++ goto err_out_shutdown_timer;
+ }
+
+ err = pdsc_start(pdsc);
+@@ -300,13 +304,14 @@ err_out_stop:
+ pdsc_stop(pdsc);
+ err_out_teardown:
+ pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
+-err_out_unmap_bars:
++err_out_shutdown_timer:
+ timer_shutdown_sync(&pdsc->wdtimer);
+ if (pdsc->wq)
+ destroy_workqueue(pdsc->wq);
+ mutex_destroy(&pdsc->config_lock);
+ mutex_destroy(&pdsc->devcmd_lock);
+ pci_free_irq_vectors(pdsc->pdev);
++err_out_unmap_bars:
+ pdsc_unmap_bars(pdsc);
+ err_out_release_regions:
+ pci_release_regions(pdsc->pdev);
+@@ -429,8 +434,6 @@ static void pdsc_remove(struct pci_dev *pdev)
+ pdsc_sriov_configure(pdev, 0);
+
+ timer_shutdown_sync(&pdsc->wdtimer);
+- if (pdsc->wq)
+- destroy_workqueue(pdsc->wq);
+
+ mutex_lock(&pdsc->config_lock);
+ set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
+@@ -438,6 +441,9 @@ static void pdsc_remove(struct pci_dev *pdev)
+ pdsc_stop(pdsc);
+ pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
+ mutex_unlock(&pdsc->config_lock);
++
++ if (pdsc->wq)
++ destroy_workqueue(pdsc->wq);
+ mutex_destroy(&pdsc->config_lock);
+ mutex_destroy(&pdsc->devcmd_lock);
+
+diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
+index 75e9cb3fc7aa66..804945ea602790 100644
+--- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
++++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
+@@ -376,9 +376,14 @@ static void xgbe_an37_set(struct xgbe_prv_data *pdata, bool enable,
+
+ XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_CTRL1, reg);
+
+- reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL);
+- reg |= XGBE_VEND2_MAC_AUTO_SW;
+- XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL, reg);
++ if (pdata->an_mode == XGBE_AN_MODE_CL37_SGMII) {
++ reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL);
++ if (enable)
++ reg |= XGBE_VEND2_MAC_AUTO_SW;
++ else
++ reg &= ~XGBE_VEND2_MAC_AUTO_SW;
++ XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL, reg);
++ }
+ }
+
+ static void xgbe_an37_restart(struct xgbe_prv_data *pdata)
+diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
+index fbf8d0ba9c4732..67b16aca565ce9 100644
+--- a/drivers/net/ethernet/cadence/macb_main.c
++++ b/drivers/net/ethernet/cadence/macb_main.c
+@@ -2582,8 +2582,26 @@ static void macb_free_consistent(struct macb *bp)
+ bp->macbgem_ops.mog_free_rx_buffers(bp);
+
+ for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+- kfree(queue->tx_skb);
+- queue->tx_skb = NULL;
++ if (queue->tx_skb) {
++ unsigned int dropped = 0, tail;
++
++ for (tail = queue->tx_tail; tail != queue->tx_head;
++ tail++) {
++ if (macb_tx_skb(queue, tail)->skb)
++ dropped++;
++ macb_tx_unmap(bp, macb_tx_skb(queue, tail), 0);
++ }
++
++ queue->stats.tx_dropped += dropped;
++ bp->dev->stats.tx_dropped += dropped;
++
++ kfree(queue->tx_skb);
++ queue->tx_skb = NULL;
++ }
++
++ queue->tx_head = 0;
++ queue->tx_tail = 0;
++
+ if (queue->tx_ring) {
+ size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
+ dma_free_coherent(&bp->pdev->dev, size,
+diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+index f56a14e09d4a3f..047a09e9e6da2d 100644
+--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
++++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+@@ -4721,6 +4721,7 @@ static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv)
+ dpaa2_mac_disconnect(mac);
+
+ dpaa2_mac_close(mac);
++ put_device(&mac->mc_dev->dev);
+ kfree(mac);
+ }
+
+diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
+index 4520c2788b1c56..8469ca27086994 100644
+--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
++++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
+@@ -1510,6 +1510,7 @@ static void dpaa2_switch_port_disconnect_mac(struct ethsw_port_priv *port_priv)
+ dpaa2_mac_disconnect(mac);
+
+ dpaa2_mac_close(mac);
++ put_device(&mac->mc_dev->dev);
+ kfree(mac);
+ }
+
+diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.c b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
+index 3088da7adf0f84..074ca626a51508 100644
+--- a/drivers/net/ethernet/freescale/fman/fman_dtsec.c
++++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
+@@ -900,22 +900,28 @@ static void dtsec_mac_config(struct phylink_config *config, unsigned int mode,
+ {
+ struct mac_device *mac_dev = fman_config_to_mac(config);
+ struct dtsec_regs __iomem *regs = mac_dev->fman_mac->regs;
+- u32 tmp;
++ u32 ecntrl, maccfg2;
++
++ maccfg2 = ioread32be(®s->maccfg2);
++ maccfg2 &= ~(MACCFG2_NIBBLE_MODE | MACCFG2_BYTE_MODE);
+
+ switch (state->interface) {
+ case PHY_INTERFACE_MODE_RMII:
+- tmp = DTSEC_ECNTRL_RMM;
++ ecntrl = DTSEC_ECNTRL_RMM;
++ maccfg2 |= MACCFG2_NIBBLE_MODE;
+ break;
+ case PHY_INTERFACE_MODE_RGMII:
+ case PHY_INTERFACE_MODE_RGMII_ID:
+ case PHY_INTERFACE_MODE_RGMII_RXID:
+ case PHY_INTERFACE_MODE_RGMII_TXID:
+- tmp = DTSEC_ECNTRL_GMIIM | DTSEC_ECNTRL_RPM;
++ ecntrl = DTSEC_ECNTRL_GMIIM | DTSEC_ECNTRL_RPM;
++ maccfg2 |= MACCFG2_BYTE_MODE;
+ break;
+ case PHY_INTERFACE_MODE_SGMII:
+ case PHY_INTERFACE_MODE_1000BASEX:
+ case PHY_INTERFACE_MODE_2500BASEX:
+- tmp = DTSEC_ECNTRL_TBIM | DTSEC_ECNTRL_SGMIIM;
++ ecntrl = DTSEC_ECNTRL_TBIM | DTSEC_ECNTRL_SGMIIM;
++ maccfg2 |= MACCFG2_BYTE_MODE;
+ break;
+ default:
+ dev_warn(mac_dev->dev, "cannot configure dTSEC for %s\n",
+@@ -923,7 +929,8 @@ static void dtsec_mac_config(struct phylink_config *config, unsigned int mode,
+ return;
+ }
+
+- iowrite32be(tmp, ®s->ecntrl);
++ iowrite32be(ecntrl, ®s->ecntrl);
++ iowrite32be(maccfg2, ®s->maccfg2);
+ }
+
+ static void dtsec_link_up(struct phylink_config *config, struct phy_device *phy,
+diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
+index 8ddd366d9fde54..e19d722e8e9a18 100644
+--- a/drivers/net/ethernet/google/gve/gve.h
++++ b/drivers/net/ethernet/google/gve/gve.h
+@@ -12,6 +12,7 @@
+ #include <linux/ethtool_netlink.h>
+ #include <linux/netdevice.h>
+ #include <linux/pci.h>
++#include <linux/timer.h>
+ #include <linux/u64_stats_sync.h>
+ #include <net/xdp.h>
+
+@@ -38,6 +39,7 @@
+
+ /* Interval to schedule a stats report update, 20000ms. */
+ #define GVE_STATS_REPORT_TIMER_PERIOD 20000
++#define GVE_RX_NAPI_RESCHED_MS 20 /* msecs */
+
+ /* Numbers of NIC tx/rx stats in stats report. */
+ #define NIC_TX_STATS_REPORT_NUM 0
+@@ -311,6 +313,7 @@ struct gve_rx_ring {
+ struct xdp_rxq_info xsk_rxq;
+ struct xsk_buff_pool *xsk_pool;
+ struct page_frag_cache page_cache; /* Page cache to allocate XDP frames */
++ struct timer_list starvation_timer; /* for queue starvation recovery */
+ };
+
+ /* A TX desc ring entry */
+diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+index 1154c1d8f66f05..3655f48e9b46ca 100644
+--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
++++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+@@ -199,14 +199,26 @@ static int gve_alloc_page_dqo(struct gve_rx_ring *rx,
+ return 0;
+ }
+
++static void gve_rx_starvation_timer(struct timer_list *t)
++{
++ struct gve_rx_ring *rx = from_timer(rx, t, starvation_timer);
++ struct gve_priv *priv = rx->gve;
++ struct gve_notify_block *block;
++
++ block = &priv->ntfy_blocks[rx->ntfy_id];
++ napi_schedule(&block->napi);
++}
++
+ static void gve_rx_free_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx)
+ {
+ struct device *hdev = &priv->pdev->dev;
+- int buf_count = rx->dqo.bufq.mask + 1;
+
+ if (rx->dqo.hdr_bufs.data) {
+- dma_free_coherent(hdev, priv->header_buf_size * buf_count,
+- rx->dqo.hdr_bufs.data, rx->dqo.hdr_bufs.addr);
++ size_t size =
++ (size_t)priv->header_buf_size * rx->dqo.num_buf_states;
++
++ dma_free_coherent(hdev, size, rx->dqo.hdr_bufs.data,
++ rx->dqo.hdr_bufs.addr);
+ rx->dqo.hdr_bufs.data = NULL;
+ }
+ }
+@@ -290,10 +302,13 @@ static void gve_rx_reset_ring_dqo(struct gve_priv *priv, int idx)
+ void gve_rx_stop_ring_dqo(struct gve_priv *priv, int idx)
+ {
+ int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
++ struct gve_rx_ring *rx = &priv->rx[idx];
+
+ if (!gve_rx_was_added_to_block(priv, idx))
+ return;
+
++ timer_shutdown_sync(&rx->starvation_timer);
++
+ gve_remove_napi(priv, ntfy_idx);
+ gve_rx_remove_from_block(priv, idx);
+ gve_rx_reset_ring_dqo(priv, idx);
+@@ -371,8 +386,10 @@ static int gve_rx_alloc_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx,
+ void gve_rx_start_ring_dqo(struct gve_priv *priv, int idx)
+ {
+ int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
++ struct gve_rx_ring *rx = &priv->rx[idx];
+
+ gve_rx_add_to_block(priv, idx);
++ timer_setup(&rx->starvation_timer, gve_rx_starvation_timer, 0);
+ gve_add_napi(priv, ntfy_idx, gve_napi_poll_dqo);
+ }
+
+@@ -406,7 +423,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
+
+ /* Allocate header buffers for header-split */
+ if (cfg->enable_header_split)
+- if (gve_rx_alloc_hdr_bufs(priv, rx, buffer_queue_slots))
++ if (gve_rx_alloc_hdr_bufs(priv, rx, rx->dqo.num_buf_states))
+ goto err;
+
+ /* Allocate RX completion queue */
+@@ -511,6 +528,7 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
+ struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq;
+ struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq;
+ struct gve_priv *priv = rx->gve;
++ u32 num_bufs_avail_to_hw;
+ u32 num_avail_slots;
+ u32 num_full_slots;
+ u32 num_posted = 0;
+@@ -541,10 +559,13 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
+ desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
+ desc->buf_addr = cpu_to_le64(buf_state->addr +
+ buf_state->page_info.page_offset);
+- if (rx->dqo.hdr_bufs.data)
++ if (rx->dqo.hdr_bufs.data) {
++ u16 buf_id = le16_to_cpu(desc->buf_id);
++
+ desc->header_buf_addr =
+ cpu_to_le64(rx->dqo.hdr_bufs.addr +
+- priv->header_buf_size * bufq->tail);
++ (size_t)priv->header_buf_size * buf_id);
++ }
+
+ bufq->tail = (bufq->tail + 1) & bufq->mask;
+ complq->num_free_slots--;
+@@ -555,6 +576,26 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
+ }
+
+ rx->fill_cnt += num_posted;
++
++ /* If the queue has fewer than GVE_RX_BUF_THRESH_DQO descriptors
++ * visible to the hardware, the hardware is in danger of starving
++ * and cannot trigger interrupts.
++ *
++ * We use a threshold of 32 because a single maximum-sized RSC
++ * packet can consume up to 19 descriptors in the Rx path. Lower
++ * thresholds (e.g., 8 or 16) would be unsafe as they could cause
++ * the device to drop/stall on a maximum-sized RSC packet.
++ *
++ * Start the timer to periodically reschedule NAPI and recover.
++ */
++ num_bufs_avail_to_hw =
++ ((bufq->tail & ~(GVE_RX_BUF_THRESH_DQO - 1)) -
++ bufq->head) & bufq->mask;
++
++ if (num_bufs_avail_to_hw < GVE_RX_BUF_THRESH_DQO) {
++ mod_timer(&rx->starvation_timer,
++ jiffies + msecs_to_jiffies(GVE_RX_NAPI_RESCHED_MS));
++ }
+ }
+
+ static void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
+@@ -791,10 +832,13 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
+ int unsplit = 0;
+
+ if (hdr_len && !hbo) {
+- rx->ctx.skb_head = gve_rx_copy_data(priv->dev, napi,
+- rx->dqo.hdr_bufs.data +
+- desc_idx * priv->header_buf_size,
+- hdr_len);
++ size_t offset =
++ (size_t)buffer_id * priv->header_buf_size;
++
++ rx->ctx.skb_head =
++ gve_rx_copy_data(priv->dev, napi,
++ rx->dqo.hdr_bufs.data + offset,
++ hdr_len);
+ if (unlikely(!rx->ctx.skb_head))
+ goto error;
+ rx->ctx.skb_tail = rx->ctx.skb_head;
+diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c
+index beb815e5289b12..035160ee8a818d 100644
+--- a/drivers/net/ethernet/hisilicon/hip04_eth.c
++++ b/drivers/net/ethernet/hisilicon/hip04_eth.c
+@@ -594,7 +594,11 @@ static int hip04_rx_poll(struct napi_struct *napi, int budget)
+ skb = build_skb(buf, priv->rx_buf_size);
+ if (unlikely(!skb)) {
+ net_dbg_ratelimited("build_skb failed\n");
+- goto refill;
++ /* Retain the slot; return budget so NAPI retries this
++ * buffer. Refill would overwrite rx_buf[]/rx_phys[]
++ * and leak them.
++ */
++ return budget;
+ }
+
+ dma_unmap_single(priv->dev, priv->rx_phys[priv->rx_head],
+@@ -622,14 +626,15 @@ static int hip04_rx_poll(struct napi_struct *napi, int budget)
+ rx++;
+ }
+
+-refill:
+ buf = netdev_alloc_frag(priv->rx_buf_size);
+ if (!buf)
+ goto done;
+ phys = dma_map_single(priv->dev, buf,
+ RX_BUF_SIZE, DMA_FROM_DEVICE);
+- if (dma_mapping_error(priv->dev, phys))
++ if (dma_mapping_error(priv->dev, phys)) {
++ skb_free_frag(buf);
+ goto done;
++ }
+ priv->rx_buf[priv->rx_head] = buf;
+ priv->rx_phys[priv->rx_head] = phys;
+ hip04_set_recv_desc(priv, phys);
+diff --git a/drivers/net/ethernet/huawei/hinic/hinic_dev.h b/drivers/net/ethernet/huawei/hinic/hinic_dev.h
+index 52ea97c818b8ec..d9ab94910a2a79 100644
+--- a/drivers/net/ethernet/huawei/hinic/hinic_dev.h
++++ b/drivers/net/ethernet/huawei/hinic/hinic_dev.h
+@@ -104,8 +104,6 @@ struct hinic_dev {
+ u16 num_rss;
+ u16 rss_limit;
+ struct hinic_rss_type rss_type;
+- u8 *rss_hkey_user;
+- s32 *rss_indir_user;
+ struct hinic_intr_coal_info *rx_intr_coalesce;
+ struct hinic_intr_coal_info *tx_intr_coalesce;
+ struct hinic_sriov_info sriov_info;
+diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c
+index c559dd4291d306..dba76a768e672a 100644
+--- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c
++++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c
+@@ -1061,17 +1061,6 @@ static int __set_rss_rxfh(struct net_device *netdev,
+ int err;
+
+ if (indir) {
+- if (!nic_dev->rss_indir_user) {
+- nic_dev->rss_indir_user =
+- kzalloc(sizeof(u32) * HINIC_RSS_INDIR_SIZE,
+- GFP_KERNEL);
+- if (!nic_dev->rss_indir_user)
+- return -ENOMEM;
+- }
+-
+- memcpy(nic_dev->rss_indir_user, indir,
+- sizeof(u32) * HINIC_RSS_INDIR_SIZE);
+-
+ err = hinic_rss_set_indir_tbl(nic_dev,
+ nic_dev->rss_tmpl_idx, indir);
+ if (err)
+@@ -1079,16 +1068,6 @@ static int __set_rss_rxfh(struct net_device *netdev,
+ }
+
+ if (key) {
+- if (!nic_dev->rss_hkey_user) {
+- nic_dev->rss_hkey_user =
+- kzalloc(HINIC_RSS_KEY_SIZE * 2, GFP_KERNEL);
+-
+- if (!nic_dev->rss_hkey_user)
+- return -ENOMEM;
+- }
+-
+- memcpy(nic_dev->rss_hkey_user, key, HINIC_RSS_KEY_SIZE);
+-
+ err = hinic_rss_set_template_tbl(nic_dev,
+ nic_dev->rss_tmpl_idx, key);
+ if (err)
+diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c
+index 32b52ecff04ffb..fa7f1e02bfafed 100644
+--- a/drivers/net/ethernet/intel/ice/ice_eswitch.c
++++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c
+@@ -506,9 +506,6 @@ int ice_eswitch_attach_vf(struct ice_pf *pf, struct ice_vf *vf)
+ struct ice_repr *repr;
+ int err;
+
+- if (!ice_is_eswitch_mode_switchdev(pf))
+- return 0;
+-
+ repr = ice_repr_create_vf(vf);
+ if (IS_ERR(repr))
+ return PTR_ERR(repr);
+diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c
+index d132eb4775513c..da1cd103a4d905 100644
+--- a/drivers/net/ethernet/intel/ice/ice_lag.c
++++ b/drivers/net/ethernet/intel/ice/ice_lag.c
+@@ -2095,7 +2095,7 @@ int ice_init_lag(struct ice_pf *pf)
+ goto free_rcp_res;
+
+ /* associate recipes to profiles */
+- for (n = 0; n < ICE_PROFID_IPV6_GTPU_IPV6_TCP_INNER; n++) {
++ for (n = 0; n < ICE_MAX_NUM_PROFILES; n++) {
+ err = ice_aq_get_recipe_to_profile(&pf->hw, n,
+ &recipe_bits, NULL);
+ if (err)
+diff --git a/drivers/net/ethernet/intel/ice/ice_parser.c b/drivers/net/ethernet/intel/ice/ice_parser.c
+index 664beb64f5570d..0c240e90a54e1a 100644
+--- a/drivers/net/ethernet/intel/ice/ice_parser.c
++++ b/drivers/net/ethernet/intel/ice/ice_parser.c
+@@ -2368,6 +2368,9 @@ int ice_parser_profile_init(struct ice_parser_result *rslt,
+ u16 proto_off = 0;
+ u16 off;
+
++ if (rslt->ptype >= ICE_FLOW_PTYPE_MAX)
++ return -EINVAL;
++
+ memset(prof, 0, sizeof(*prof));
+ set_bit(rslt->ptype, prof->ptypes);
+ if (blk == ICE_BLK_SW) {
+diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c
+index 17d9d2d8ea47ca..ca0d4ba2b63b2e 100644
+--- a/drivers/net/ethernet/intel/ice/ice_ptp.c
++++ b/drivers/net/ethernet/intel/ice/ice_ptp.c
+@@ -514,7 +514,7 @@ static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp)
+ return 0;
+ }
+
+- return ice_ptp_extend_32b_ts(pf->ptp.cached_phc_time,
++ return ice_ptp_extend_32b_ts(READ_ONCE(pf->ptp.cached_phc_time),
+ (in_tstamp >> 8) & mask);
+ }
+
+@@ -2886,6 +2886,11 @@ void ice_ptp_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
+ struct ice_ptp *ptp = &pf->ptp;
+ int err;
+
++ if (ptp->state == ICE_PTP_UNINIT) {
++ dev_dbg(ice_pf_to_dev(pf), "PTP was not initialized, skipping rebuild\n");
++ return;
++ }
++
+ if (ptp->state == ICE_PTP_READY) {
+ ice_ptp_prepare_for_reset(pf, reset_type);
+ } else if (ptp->state != ICE_PTP_RESETTING) {
+diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c
+index 8aabf7749aa5e0..d33a761cfdc04e 100644
+--- a/drivers/net/ethernet/intel/ice/ice_sriov.c
++++ b/drivers/net/ethernet/intel/ice/ice_sriov.c
+@@ -596,12 +596,14 @@ static int ice_start_vfs(struct ice_pf *pf)
+ goto teardown;
+ }
+
+- retval = ice_eswitch_attach_vf(pf, vf);
+- if (retval) {
+- dev_err(ice_pf_to_dev(pf), "Failed to attach VF %d to eswitch, error %d",
+- vf->vf_id, retval);
+- ice_vf_vsi_release(vf);
+- goto teardown;
++ if (ice_is_eswitch_mode_switchdev(pf)) {
++ retval = ice_eswitch_attach_vf(pf, vf);
++ if (retval) {
++ dev_err(ice_pf_to_dev(pf), "Failed to attach VF %d to eswitch, error %d",
++ vf->vf_id, retval);
++ ice_vf_vsi_release(vf);
++ goto teardown;
++ }
+ }
+
+ set_bit(ICE_VF_STATE_INIT, vf->vf_states);
+diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
+index afa9e7ce463763..71530ae79a87e8 100644
+--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c
++++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
+@@ -811,7 +811,8 @@ void ice_reset_all_vfs(struct ice_pf *pf)
+ }
+ ice_vf_post_vsi_rebuild(vf);
+
+- ice_eswitch_attach_vf(pf, vf);
++ if (ice_is_eswitch_mode_switchdev(pf))
++ ice_eswitch_attach_vf(pf, vf);
+
+ mutex_unlock(&vf->cfg_lock);
+ }
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+index df662d07a5e9c7..f64171359ec8c6 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+@@ -440,7 +440,7 @@ struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)
+ return &rvu->pf[rvu_get_pf(pcifunc)];
+ }
+
+-static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
++bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
+ {
+ int pf, vf, nvfs;
+ u64 cfg;
+@@ -2401,7 +2401,7 @@ static inline void rvu_afvf_mbox_up_handler(struct work_struct *work)
+ __rvu_mbox_up_handler(mwork, TYPE_AFVF);
+ }
+
+-static int rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr,
++static int rvu_get_mbox_regions(struct rvu *rvu, void __iomem **mbox_addr,
+ int num, int type, unsigned long *pf_bmap)
+ {
+ struct rvu_hwinfo *hw = rvu->hw;
+@@ -2426,7 +2426,7 @@ static int rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr,
+ bar4 = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR);
+ bar4 += region * MBOX_SIZE;
+ }
+- mbox_addr[region] = (void *)ioremap_wc(bar4, MBOX_SIZE);
++ mbox_addr[region] = ioremap_wc(bar4, MBOX_SIZE);
+ if (!mbox_addr[region])
+ goto error;
+ }
+@@ -2449,7 +2449,7 @@ static int rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr,
+ RVU_AF_PF_BAR4_ADDR);
+ bar4 += region * MBOX_SIZE;
+ }
+- mbox_addr[region] = (void *)ioremap_wc(bar4, MBOX_SIZE);
++ mbox_addr[region] = ioremap_wc(bar4, MBOX_SIZE);
+ if (!mbox_addr[region])
+ goto error;
+ }
+@@ -2457,7 +2457,7 @@ static int rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr,
+
+ error:
+ while (region--)
+- iounmap((void __iomem *)mbox_addr[region]);
++ iounmap(mbox_addr[region]);
+ return -ENOMEM;
+ }
+
+@@ -2467,10 +2467,10 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
+ void (mbox_up_handler)(struct work_struct *))
+ {
+ int err = -EINVAL, i, dir, dir_up;
++ void __iomem **mbox_regions;
+ void __iomem *reg_base;
+ struct rvu_work *mwork;
+ unsigned long *pf_bmap;
+- void **mbox_regions;
+ const char *name;
+ u64 cfg;
+
+@@ -2493,7 +2493,7 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
+
+ mutex_init(&rvu->mbox_lock);
+
+- mbox_regions = kcalloc(num, sizeof(void *), GFP_KERNEL);
++ mbox_regions = kcalloc(num, sizeof(void __iomem *), GFP_KERNEL);
+ if (!mbox_regions) {
+ err = -ENOMEM;
+ goto free_bitmap;
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+index 77a03e29a77116..34b2a16e88446b 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+@@ -825,6 +825,7 @@ int rvu_get_pf(u16 pcifunc);
+ struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc);
+ void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf);
+ bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr);
++bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc);
+ bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype);
+ int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot);
+ int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf);
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c
+index 4a3370a40dd887..eee467c5bc89ce 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c
+@@ -178,6 +178,15 @@ int rvu_mbox_handler_lmtst_tbl_setup(struct rvu *rvu,
+ * pcifunc (will be the one who is calling this mailbox).
+ */
+ if (req->base_pcifunc) {
++ /* A VF is untrusted and must not redirect its LMTLINE to
++ * another PF's region, so confine VF callers to their own PF.
++ */
++ if (is_vf(req->hdr.pcifunc) &&
++ (!is_pf_func_valid(rvu, req->base_pcifunc) ||
++ rvu_get_pf(req->hdr.pcifunc) !=
++ rvu_get_pf(req->base_pcifunc)))
++ return -EPERM;
++
+ /* Calculating the LMT table index equivalent to primary
+ * pcifunc.
+ */
+diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
+index 2750326bfcf8b1..a9057711b62255 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
++++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
+@@ -151,6 +151,7 @@ exit:
+ if (allocated) {
+ pfvf->flags |= OTX2_FLAG_MCAM_ENTRIES_ALLOC;
+ pfvf->flags |= OTX2_FLAG_NTUPLE_SUPPORT;
++ pfvf->flags |= OTX2_FLAG_TC_FLOWER_SUPPORT;
+ }
+
+ if (allocated != count)
+diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+index b0b0b4929e8ce2..c314139e0d7818 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
++++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+@@ -593,8 +593,7 @@ static int otx2_pfvf_mbox_init(struct otx2_nic *pf, int numvfs)
+ base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) +
+ MBOX_SIZE;
+ else
+- base = readq((void __iomem *)((u64)pf->reg_base +
+- RVU_PF_VF_BAR4_ADDR));
++ base = readq(pf->reg_base + RVU_PF_VF_BAR4_ADDR);
+
+ hwbase = ioremap_wc(base, MBOX_SIZE * pf->total_vfs);
+ if (!hwbase) {
+@@ -1024,6 +1023,9 @@ static int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
+ char *irq_name;
+ int err;
+
++ /* Clear stale mailbox interrupt state before installing the handler. */
++ otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
++
+ /* Register mailbox interrupt handler */
+ irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE];
+ snprintf(irq_name, NAME_SIZE, "RVUPFAF Mbox");
+@@ -1035,10 +1037,7 @@ static int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
+ return err;
+ }
+
+- /* Enable mailbox interrupt for msgs coming from AF.
+- * First clear to avoid spurious interrupts, if any.
+- */
+- otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
++ /* Enable mailbox interrupt for msgs coming from AF. */
+ otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0));
+
+ if (!probe_af)
+@@ -1439,13 +1438,13 @@ static void otx2_free_sq_res(struct otx2_nic *pf)
+ otx2_sq_free_sqbs(pf);
+ for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
+ sq = &qset->sq[qidx];
+- /* Skip freeing Qos queues if they are not initialized */
+- if (!sq->sqe)
+- continue;
+- qmem_free(pf->dev, sq->sqe);
+- qmem_free(pf->dev, sq->tso_hdrs);
+- qmem_free(pf->dev, sq->timestamps);
+- kfree(sq->sg);
++ /* sq->sqe is not initialized for unused QoS queues */
++ if (sq->sqe) {
++ qmem_free(pf->dev, sq->sqe);
++ qmem_free(pf->dev, sq->tso_hdrs);
++ qmem_free(pf->dev, sq->timestamps);
++ kfree(sq->sg);
++ }
+ kfree(sq->sqb_ptrs);
+ }
+ }
+@@ -1571,13 +1570,12 @@ static int otx2_init_hw_resources(struct otx2_nic *pf)
+ return err;
+
+ err_free_nix_queues:
+- otx2_free_sq_res(pf);
+ otx2_free_cq_res(pf);
+ otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
+ err_free_txsch:
+ otx2_txschq_stop(pf);
+ err_free_sq_ptrs:
+- otx2_sq_free_sqbs(pf);
++ otx2_free_sq_res(pf);
+ err_free_rq_ptrs:
+ otx2_free_aura_ptr(pf, AURA_NIX_RQ);
+ otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
+diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
+index ed041c3f714f26..b26fc848430f50 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
++++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
+@@ -30,6 +30,7 @@
+ #define OTX2_UNSUPP_LSE_DEPTH GENMASK(6, 4)
+
+ #define MCAST_INVALID_GRP (-1U)
++#define RATE_MANTISSA_BITS 8
+
+ struct otx2_tc_flow_stats {
+ u64 bytes;
+@@ -90,28 +91,30 @@ static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst,
+ static void otx2_get_egress_rate_cfg(u64 maxrate, u32 *exp,
+ u32 *mantissa, u32 *div_exp)
+ {
+- u64 tmp;
+-
+ /* Rate calculation by hardware
+ *
+ * PIR_ADD = ((256 + mantissa) << exp) / 256
+ * rate = (2 * PIR_ADD) / ( 1 << div_exp)
+ * The resultant rate is in Mbps.
++ *
++ * Use div_exp = 0 and compute exp/mantissa for maxrate / 2; the
++ * leading factor of two yields the full rate. Rates below 2 Mbps
++ * are floored to the smallest step (exp = 0, mantissa = 0).
+ */
+
+- /* 2Mbps to 100Gbps can be expressed with div_exp = 0.
+- * Setting this to '0' will ease the calculation of
+- * exponent and mantissa.
+- */
+ *div_exp = 0;
+-
+ if (maxrate) {
+- *exp = ilog2(maxrate) ? ilog2(maxrate) - 1 : 0;
+- tmp = maxrate - rounddown_pow_of_two(maxrate);
+- if (maxrate < MAX_RATE_MANTISSA)
+- *mantissa = tmp * 2;
+- else
+- *mantissa = tmp / (1ULL << (*exp - 7));
++ maxrate = maxrate / 2;
++ if (!maxrate) {
++ /* Rates below 2 Mbps map to the smallest step */
++ *exp = 0;
++ *mantissa = 0;
++ } else {
++ *exp = ilog2(maxrate);
++ /* Clear MSB and derive fractional bits */
++ maxrate &= ~BIT(*exp);
++ *mantissa = (maxrate << RATE_MANTISSA_BITS) >> *exp;
++ }
+ } else {
+ /* Instead of disabling rate limiting, set all values to max */
+ *exp = MAX_RATE_EXPONENT;
+diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
+index 99fcc5661674be..e26c830ae1ffc7 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
++++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
+@@ -245,9 +245,15 @@ static int otx2vf_register_mbox_intr(struct otx2_nic *vf, bool probe_pf)
+ {
+ struct otx2_hw *hw = &vf->hw;
+ struct msg_req *req;
++ u64 mbox_int_mask;
+ char *irq_name;
+ int err;
+
++ mbox_int_mask = BIT_ULL(0);
++
++ /* Clear stale mailbox interrupt state before installing the handler. */
++ otx2_write64(vf, RVU_VF_INT, mbox_int_mask);
++
+ /* Register mailbox interrupt handler */
+ irq_name = &hw->irq_name[RVU_VF_INT_VEC_MBOX * NAME_SIZE];
+ snprintf(irq_name, NAME_SIZE, "RVUVFAF Mbox");
+@@ -259,11 +265,8 @@ static int otx2vf_register_mbox_intr(struct otx2_nic *vf, bool probe_pf)
+ return err;
+ }
+
+- /* Enable mailbox interrupt for msgs coming from PF.
+- * First clear to avoid spurious interrupts, if any.
+- */
+- otx2_write64(vf, RVU_VF_INT, BIT_ULL(0));
+- otx2_write64(vf, RVU_VF_INT_ENA_W1S, BIT_ULL(0));
++ /* Enable mailbox interrupt for msgs coming from PF. */
++ otx2_write64(vf, RVU_VF_INT_ENA_W1S, mbox_int_mask);
+
+ if (!probe_pf)
+ return 0;
+diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig
+index 95c4405b7d7bee..7bfd3f230ff507 100644
+--- a/drivers/net/ethernet/mediatek/Kconfig
++++ b/drivers/net/ethernet/mediatek/Kconfig
+@@ -7,14 +7,6 @@ config NET_VENDOR_MEDIATEK
+
+ if NET_VENDOR_MEDIATEK
+
+-config NET_AIROHA
+- tristate "Airoha SoC Gigabit Ethernet support"
+- depends on NET_DSA || !NET_DSA
+- select PAGE_POOL
+- help
+- This driver supports the gigabit ethernet MACs in the
+- Airoha SoC family.
+-
+ config NET_MEDIATEK_SOC_WED
+ depends on ARCH_MEDIATEK || COMPILE_TEST
+ def_bool NET_MEDIATEK_SOC != n
+diff --git a/drivers/net/ethernet/mediatek/Makefile b/drivers/net/ethernet/mediatek/Makefile
+index ddbb7f4a516cac..03e008fbc859b3 100644
+--- a/drivers/net/ethernet/mediatek/Makefile
++++ b/drivers/net/ethernet/mediatek/Makefile
+@@ -11,4 +11,3 @@ mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_debugfs.o
+ endif
+ obj-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_ops.o
+ obj-$(CONFIG_NET_MEDIATEK_STAR_EMAC) += mtk_star_emac.o
+-obj-$(CONFIG_NET_AIROHA) += airoha_eth.o
+diff --git a/drivers/net/ethernet/mediatek/airoha_eth.c b/drivers/net/ethernet/mediatek/airoha_eth.c
+deleted file mode 100644
+index 07f89e08b5faad..00000000000000
+--- a/drivers/net/ethernet/mediatek/airoha_eth.c
++++ /dev/null
+@@ -1,3060 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0-only
+-/*
+- * Copyright (c) 2024 AIROHA Inc
+- * Author: Lorenzo Bianconi <[email protected]>
+- */
+-#include <linux/etherdevice.h>
+-#include <linux/iopoll.h>
+-#include <linux/kernel.h>
+-#include <linux/netdevice.h>
+-#include <linux/of.h>
+-#include <linux/of_net.h>
+-#include <linux/platform_device.h>
+-#include <linux/reset.h>
+-#include <linux/tcp.h>
+-#include <linux/u64_stats_sync.h>
+-#include <net/dsa.h>
+-#include <net/page_pool/helpers.h>
+-#include <net/pkt_cls.h>
+-#include <uapi/linux/ppp_defs.h>
+-
+-#define AIROHA_MAX_NUM_GDM_PORTS 1
+-#define AIROHA_MAX_NUM_QDMA 2
+-#define AIROHA_MAX_NUM_RSTS 3
+-#define AIROHA_MAX_NUM_XSI_RSTS 5
+-#define AIROHA_MAX_MTU 2000
+-#define AIROHA_MAX_PACKET_SIZE 2048
+-#define AIROHA_NUM_QOS_CHANNELS 4
+-#define AIROHA_NUM_QOS_QUEUES 8
+-#define AIROHA_NUM_TX_RING 32
+-#define AIROHA_NUM_RX_RING 32
+-#define AIROHA_FE_MC_MAX_VLAN_TABLE 64
+-#define AIROHA_FE_MC_MAX_VLAN_PORT 16
+-#define AIROHA_NUM_TX_IRQ 2
+-#define HW_DSCP_NUM 2048
+-#define IRQ_QUEUE_LEN(_n) ((_n) ? 1024 : 2048)
+-#define TX_DSCP_NUM 1024
+-#define RX_DSCP_NUM(_n) \
+- ((_n) == 2 ? 128 : \
+- (_n) == 11 ? 128 : \
+- (_n) == 15 ? 128 : \
+- (_n) == 0 ? 1024 : 16)
+-
+-#define PSE_RSV_PAGES 128
+-#define PSE_QUEUE_RSV_PAGES 64
+-
+-/* FE */
+-#define PSE_BASE 0x0100
+-#define CSR_IFC_BASE 0x0200
+-#define CDM1_BASE 0x0400
+-#define GDM1_BASE 0x0500
+-#define PPE1_BASE 0x0c00
+-
+-#define CDM2_BASE 0x1400
+-#define GDM2_BASE 0x1500
+-
+-#define GDM3_BASE 0x1100
+-#define GDM4_BASE 0x2500
+-
+-#define GDM_BASE(_n) \
+- ((_n) == 4 ? GDM4_BASE : \
+- (_n) == 3 ? GDM3_BASE : \
+- (_n) == 2 ? GDM2_BASE : GDM1_BASE)
+-
+-#define REG_FE_DMA_GLO_CFG 0x0000
+-#define FE_DMA_GLO_L2_SPACE_MASK GENMASK(7, 4)
+-#define FE_DMA_GLO_PG_SZ_MASK BIT(3)
+-
+-#define REG_FE_RST_GLO_CFG 0x0004
+-#define FE_RST_GDM4_MBI_ARB_MASK BIT(3)
+-#define FE_RST_GDM3_MBI_ARB_MASK BIT(2)
+-#define FE_RST_CORE_MASK BIT(0)
+-
+-#define REG_FE_WAN_MAC_H 0x0030
+-#define REG_FE_LAN_MAC_H 0x0040
+-
+-#define REG_FE_MAC_LMIN(_n) ((_n) + 0x04)
+-#define REG_FE_MAC_LMAX(_n) ((_n) + 0x08)
+-
+-#define REG_FE_CDM1_OQ_MAP0 0x0050
+-#define REG_FE_CDM1_OQ_MAP1 0x0054
+-#define REG_FE_CDM1_OQ_MAP2 0x0058
+-#define REG_FE_CDM1_OQ_MAP3 0x005c
+-
+-#define REG_FE_PCE_CFG 0x0070
+-#define PCE_DPI_EN_MASK BIT(2)
+-#define PCE_KA_EN_MASK BIT(1)
+-#define PCE_MC_EN_MASK BIT(0)
+-
+-#define REG_FE_PSE_QUEUE_CFG_WR 0x0080
+-#define PSE_CFG_PORT_ID_MASK GENMASK(27, 24)
+-#define PSE_CFG_QUEUE_ID_MASK GENMASK(20, 16)
+-#define PSE_CFG_WR_EN_MASK BIT(8)
+-#define PSE_CFG_OQRSV_SEL_MASK BIT(0)
+-
+-#define REG_FE_PSE_QUEUE_CFG_VAL 0x0084
+-#define PSE_CFG_OQ_RSV_MASK GENMASK(13, 0)
+-
+-#define PSE_FQ_CFG 0x008c
+-#define PSE_FQ_LIMIT_MASK GENMASK(14, 0)
+-
+-#define REG_FE_PSE_BUF_SET 0x0090
+-#define PSE_SHARE_USED_LTHD_MASK GENMASK(31, 16)
+-#define PSE_ALLRSV_MASK GENMASK(14, 0)
+-
+-#define REG_PSE_SHARE_USED_THD 0x0094
+-#define PSE_SHARE_USED_MTHD_MASK GENMASK(31, 16)
+-#define PSE_SHARE_USED_HTHD_MASK GENMASK(15, 0)
+-
+-#define REG_GDM_MISC_CFG 0x0148
+-#define GDM2_RDM_ACK_WAIT_PREF_MASK BIT(9)
+-#define GDM2_CHN_VLD_MODE_MASK BIT(5)
+-
+-#define REG_FE_CSR_IFC_CFG CSR_IFC_BASE
+-#define FE_IFC_EN_MASK BIT(0)
+-
+-#define REG_FE_VIP_PORT_EN 0x01f0
+-#define REG_FE_IFC_PORT_EN 0x01f4
+-
+-#define REG_PSE_IQ_REV1 (PSE_BASE + 0x08)
+-#define PSE_IQ_RES1_P2_MASK GENMASK(23, 16)
+-
+-#define REG_PSE_IQ_REV2 (PSE_BASE + 0x0c)
+-#define PSE_IQ_RES2_P5_MASK GENMASK(15, 8)
+-#define PSE_IQ_RES2_P4_MASK GENMASK(7, 0)
+-
+-#define REG_FE_VIP_EN(_n) (0x0300 + ((_n) << 3))
+-#define PATN_FCPU_EN_MASK BIT(7)
+-#define PATN_SWP_EN_MASK BIT(6)
+-#define PATN_DP_EN_MASK BIT(5)
+-#define PATN_SP_EN_MASK BIT(4)
+-#define PATN_TYPE_MASK GENMASK(3, 1)
+-#define PATN_EN_MASK BIT(0)
+-
+-#define REG_FE_VIP_PATN(_n) (0x0304 + ((_n) << 3))
+-#define PATN_DP_MASK GENMASK(31, 16)
+-#define PATN_SP_MASK GENMASK(15, 0)
+-
+-#define REG_CDM1_VLAN_CTRL CDM1_BASE
+-#define CDM1_VLAN_MASK GENMASK(31, 16)
+-
+-#define REG_CDM1_FWD_CFG (CDM1_BASE + 0x08)
+-#define CDM1_VIP_QSEL_MASK GENMASK(24, 20)
+-
+-#define REG_CDM1_CRSN_QSEL(_n) (CDM1_BASE + 0x10 + ((_n) << 2))
+-#define CDM1_CRSN_QSEL_REASON_MASK(_n) \
+- GENMASK(4 + (((_n) % 4) << 3), (((_n) % 4) << 3))
+-
+-#define REG_CDM2_FWD_CFG (CDM2_BASE + 0x08)
+-#define CDM2_OAM_QSEL_MASK GENMASK(31, 27)
+-#define CDM2_VIP_QSEL_MASK GENMASK(24, 20)
+-
+-#define REG_CDM2_CRSN_QSEL(_n) (CDM2_BASE + 0x10 + ((_n) << 2))
+-#define CDM2_CRSN_QSEL_REASON_MASK(_n) \
+- GENMASK(4 + (((_n) % 4) << 3), (((_n) % 4) << 3))
+-
+-#define REG_GDM_FWD_CFG(_n) GDM_BASE(_n)
+-#define GDM_DROP_CRC_ERR BIT(23)
+-#define GDM_IP4_CKSUM BIT(22)
+-#define GDM_TCP_CKSUM BIT(21)
+-#define GDM_UDP_CKSUM BIT(20)
+-#define GDM_UCFQ_MASK GENMASK(15, 12)
+-#define GDM_BCFQ_MASK GENMASK(11, 8)
+-#define GDM_MCFQ_MASK GENMASK(7, 4)
+-#define GDM_OCFQ_MASK GENMASK(3, 0)
+-
+-#define REG_GDM_INGRESS_CFG(_n) (GDM_BASE(_n) + 0x10)
+-#define GDM_INGRESS_FC_EN_MASK BIT(1)
+-#define GDM_STAG_EN_MASK BIT(0)
+-
+-#define REG_GDM_LEN_CFG(_n) (GDM_BASE(_n) + 0x14)
+-#define GDM_SHORT_LEN_MASK GENMASK(13, 0)
+-#define GDM_LONG_LEN_MASK GENMASK(29, 16)
+-
+-#define REG_FE_CPORT_CFG (GDM1_BASE + 0x40)
+-#define FE_CPORT_PAD BIT(26)
+-#define FE_CPORT_PORT_XFC_MASK BIT(25)
+-#define FE_CPORT_QUEUE_XFC_MASK BIT(24)
+-
+-#define REG_FE_GDM_MIB_CLEAR(_n) (GDM_BASE(_n) + 0xf0)
+-#define FE_GDM_MIB_RX_CLEAR_MASK BIT(1)
+-#define FE_GDM_MIB_TX_CLEAR_MASK BIT(0)
+-
+-#define REG_FE_GDM1_MIB_CFG (GDM1_BASE + 0xf4)
+-#define FE_STRICT_RFC2819_MODE_MASK BIT(31)
+-#define FE_GDM1_TX_MIB_SPLIT_EN_MASK BIT(17)
+-#define FE_GDM1_RX_MIB_SPLIT_EN_MASK BIT(16)
+-#define FE_TX_MIB_ID_MASK GENMASK(15, 8)
+-#define FE_RX_MIB_ID_MASK GENMASK(7, 0)
+-
+-#define REG_FE_GDM_TX_OK_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x104)
+-#define REG_FE_GDM_TX_OK_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x10c)
+-#define REG_FE_GDM_TX_ETH_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x110)
+-#define REG_FE_GDM_TX_ETH_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x114)
+-#define REG_FE_GDM_TX_ETH_DROP_CNT(_n) (GDM_BASE(_n) + 0x118)
+-#define REG_FE_GDM_TX_ETH_BC_CNT(_n) (GDM_BASE(_n) + 0x11c)
+-#define REG_FE_GDM_TX_ETH_MC_CNT(_n) (GDM_BASE(_n) + 0x120)
+-#define REG_FE_GDM_TX_ETH_RUNT_CNT(_n) (GDM_BASE(_n) + 0x124)
+-#define REG_FE_GDM_TX_ETH_LONG_CNT(_n) (GDM_BASE(_n) + 0x128)
+-#define REG_FE_GDM_TX_ETH_E64_CNT_L(_n) (GDM_BASE(_n) + 0x12c)
+-#define REG_FE_GDM_TX_ETH_L64_CNT_L(_n) (GDM_BASE(_n) + 0x130)
+-#define REG_FE_GDM_TX_ETH_L127_CNT_L(_n) (GDM_BASE(_n) + 0x134)
+-#define REG_FE_GDM_TX_ETH_L255_CNT_L(_n) (GDM_BASE(_n) + 0x138)
+-#define REG_FE_GDM_TX_ETH_L511_CNT_L(_n) (GDM_BASE(_n) + 0x13c)
+-#define REG_FE_GDM_TX_ETH_L1023_CNT_L(_n) (GDM_BASE(_n) + 0x140)
+-
+-#define REG_FE_GDM_RX_OK_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x148)
+-#define REG_FE_GDM_RX_FC_DROP_CNT(_n) (GDM_BASE(_n) + 0x14c)
+-#define REG_FE_GDM_RX_RC_DROP_CNT(_n) (GDM_BASE(_n) + 0x150)
+-#define REG_FE_GDM_RX_OVERFLOW_DROP_CNT(_n) (GDM_BASE(_n) + 0x154)
+-#define REG_FE_GDM_RX_ERROR_DROP_CNT(_n) (GDM_BASE(_n) + 0x158)
+-#define REG_FE_GDM_RX_OK_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x15c)
+-#define REG_FE_GDM_RX_ETH_PKT_CNT_L(_n) (GDM_BASE(_n) + 0x160)
+-#define REG_FE_GDM_RX_ETH_BYTE_CNT_L(_n) (GDM_BASE(_n) + 0x164)
+-#define REG_FE_GDM_RX_ETH_DROP_CNT(_n) (GDM_BASE(_n) + 0x168)
+-#define REG_FE_GDM_RX_ETH_BC_CNT(_n) (GDM_BASE(_n) + 0x16c)
+-#define REG_FE_GDM_RX_ETH_MC_CNT(_n) (GDM_BASE(_n) + 0x170)
+-#define REG_FE_GDM_RX_ETH_CRC_ERR_CNT(_n) (GDM_BASE(_n) + 0x174)
+-#define REG_FE_GDM_RX_ETH_FRAG_CNT(_n) (GDM_BASE(_n) + 0x178)
+-#define REG_FE_GDM_RX_ETH_JABBER_CNT(_n) (GDM_BASE(_n) + 0x17c)
+-#define REG_FE_GDM_RX_ETH_RUNT_CNT(_n) (GDM_BASE(_n) + 0x180)
+-#define REG_FE_GDM_RX_ETH_LONG_CNT(_n) (GDM_BASE(_n) + 0x184)
+-#define REG_FE_GDM_RX_ETH_E64_CNT_L(_n) (GDM_BASE(_n) + 0x188)
+-#define REG_FE_GDM_RX_ETH_L64_CNT_L(_n) (GDM_BASE(_n) + 0x18c)
+-#define REG_FE_GDM_RX_ETH_L127_CNT_L(_n) (GDM_BASE(_n) + 0x190)
+-#define REG_FE_GDM_RX_ETH_L255_CNT_L(_n) (GDM_BASE(_n) + 0x194)
+-#define REG_FE_GDM_RX_ETH_L511_CNT_L(_n) (GDM_BASE(_n) + 0x198)
+-#define REG_FE_GDM_RX_ETH_L1023_CNT_L(_n) (GDM_BASE(_n) + 0x19c)
+-
+-#define REG_PPE1_TB_HASH_CFG (PPE1_BASE + 0x250)
+-#define PPE1_SRAM_TABLE_EN_MASK BIT(0)
+-#define PPE1_SRAM_HASH1_EN_MASK BIT(8)
+-#define PPE1_DRAM_TABLE_EN_MASK BIT(16)
+-#define PPE1_DRAM_HASH1_EN_MASK BIT(24)
+-
+-#define REG_FE_GDM_TX_OK_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x280)
+-#define REG_FE_GDM_TX_OK_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x284)
+-#define REG_FE_GDM_TX_ETH_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x288)
+-#define REG_FE_GDM_TX_ETH_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x28c)
+-
+-#define REG_FE_GDM_RX_OK_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x290)
+-#define REG_FE_GDM_RX_OK_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x294)
+-#define REG_FE_GDM_RX_ETH_PKT_CNT_H(_n) (GDM_BASE(_n) + 0x298)
+-#define REG_FE_GDM_RX_ETH_BYTE_CNT_H(_n) (GDM_BASE(_n) + 0x29c)
+-#define REG_FE_GDM_TX_ETH_E64_CNT_H(_n) (GDM_BASE(_n) + 0x2b8)
+-#define REG_FE_GDM_TX_ETH_L64_CNT_H(_n) (GDM_BASE(_n) + 0x2bc)
+-#define REG_FE_GDM_TX_ETH_L127_CNT_H(_n) (GDM_BASE(_n) + 0x2c0)
+-#define REG_FE_GDM_TX_ETH_L255_CNT_H(_n) (GDM_BASE(_n) + 0x2c4)
+-#define REG_FE_GDM_TX_ETH_L511_CNT_H(_n) (GDM_BASE(_n) + 0x2c8)
+-#define REG_FE_GDM_TX_ETH_L1023_CNT_H(_n) (GDM_BASE(_n) + 0x2cc)
+-#define REG_FE_GDM_RX_ETH_E64_CNT_H(_n) (GDM_BASE(_n) + 0x2e8)
+-#define REG_FE_GDM_RX_ETH_L64_CNT_H(_n) (GDM_BASE(_n) + 0x2ec)
+-#define REG_FE_GDM_RX_ETH_L127_CNT_H(_n) (GDM_BASE(_n) + 0x2f0)
+-#define REG_FE_GDM_RX_ETH_L255_CNT_H(_n) (GDM_BASE(_n) + 0x2f4)
+-#define REG_FE_GDM_RX_ETH_L511_CNT_H(_n) (GDM_BASE(_n) + 0x2f8)
+-#define REG_FE_GDM_RX_ETH_L1023_CNT_H(_n) (GDM_BASE(_n) + 0x2fc)
+-
+-#define REG_GDM2_CHN_RLS (GDM2_BASE + 0x20)
+-#define MBI_RX_AGE_SEL_MASK GENMASK(26, 25)
+-#define MBI_TX_AGE_SEL_MASK GENMASK(18, 17)
+-
+-#define REG_GDM3_FWD_CFG GDM3_BASE
+-#define GDM3_PAD_EN_MASK BIT(28)
+-
+-#define REG_GDM4_FWD_CFG GDM4_BASE
+-#define GDM4_PAD_EN_MASK BIT(28)
+-#define GDM4_SPORT_OFFSET0_MASK GENMASK(11, 8)
+-
+-#define REG_GDM4_SRC_PORT_SET (GDM4_BASE + 0x23c)
+-#define GDM4_SPORT_OFF2_MASK GENMASK(19, 16)
+-#define GDM4_SPORT_OFF1_MASK GENMASK(15, 12)
+-#define GDM4_SPORT_OFF0_MASK GENMASK(11, 8)
+-
+-#define REG_IP_FRAG_FP 0x2010
+-#define IP_ASSEMBLE_PORT_MASK GENMASK(24, 21)
+-#define IP_ASSEMBLE_NBQ_MASK GENMASK(20, 16)
+-#define IP_FRAGMENT_PORT_MASK GENMASK(8, 5)
+-#define IP_FRAGMENT_NBQ_MASK GENMASK(4, 0)
+-
+-#define REG_MC_VLAN_EN 0x2100
+-#define MC_VLAN_EN_MASK BIT(0)
+-
+-#define REG_MC_VLAN_CFG 0x2104
+-#define MC_VLAN_CFG_CMD_DONE_MASK BIT(31)
+-#define MC_VLAN_CFG_TABLE_ID_MASK GENMASK(21, 16)
+-#define MC_VLAN_CFG_PORT_ID_MASK GENMASK(11, 8)
+-#define MC_VLAN_CFG_TABLE_SEL_MASK BIT(4)
+-#define MC_VLAN_CFG_RW_MASK BIT(0)
+-
+-#define REG_MC_VLAN_DATA 0x2108
+-
+-#define REG_CDM5_RX_OQ1_DROP_CNT 0x29d4
+-
+-/* QDMA */
+-#define REG_QDMA_GLOBAL_CFG 0x0004
+-#define GLOBAL_CFG_RX_2B_OFFSET_MASK BIT(31)
+-#define GLOBAL_CFG_DMA_PREFERENCE_MASK GENMASK(30, 29)
+-#define GLOBAL_CFG_CPU_TXR_RR_MASK BIT(28)
+-#define GLOBAL_CFG_DSCP_BYTE_SWAP_MASK BIT(27)
+-#define GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK BIT(26)
+-#define GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK BIT(25)
+-#define GLOBAL_CFG_OAM_MODIFY_MASK BIT(24)
+-#define GLOBAL_CFG_RESET_MASK BIT(23)
+-#define GLOBAL_CFG_RESET_DONE_MASK BIT(22)
+-#define GLOBAL_CFG_MULTICAST_EN_MASK BIT(21)
+-#define GLOBAL_CFG_IRQ1_EN_MASK BIT(20)
+-#define GLOBAL_CFG_IRQ0_EN_MASK BIT(19)
+-#define GLOBAL_CFG_LOOPCNT_EN_MASK BIT(18)
+-#define GLOBAL_CFG_RD_BYPASS_WR_MASK BIT(17)
+-#define GLOBAL_CFG_QDMA_LOOPBACK_MASK BIT(16)
+-#define GLOBAL_CFG_LPBK_RXQ_SEL_MASK GENMASK(13, 8)
+-#define GLOBAL_CFG_CHECK_DONE_MASK BIT(7)
+-#define GLOBAL_CFG_TX_WB_DONE_MASK BIT(6)
+-#define GLOBAL_CFG_MAX_ISSUE_NUM_MASK GENMASK(5, 4)
+-#define GLOBAL_CFG_RX_DMA_BUSY_MASK BIT(3)
+-#define GLOBAL_CFG_RX_DMA_EN_MASK BIT(2)
+-#define GLOBAL_CFG_TX_DMA_BUSY_MASK BIT(1)
+-#define GLOBAL_CFG_TX_DMA_EN_MASK BIT(0)
+-
+-#define REG_FWD_DSCP_BASE 0x0010
+-#define REG_FWD_BUF_BASE 0x0014
+-
+-#define REG_HW_FWD_DSCP_CFG 0x0018
+-#define HW_FWD_DSCP_PAYLOAD_SIZE_MASK GENMASK(29, 28)
+-#define HW_FWD_DSCP_SCATTER_LEN_MASK GENMASK(17, 16)
+-#define HW_FWD_DSCP_MIN_SCATTER_LEN_MASK GENMASK(15, 0)
+-
+-#define REG_INT_STATUS(_n) \
+- (((_n) == 4) ? 0x0730 : \
+- ((_n) == 3) ? 0x0724 : \
+- ((_n) == 2) ? 0x0720 : \
+- ((_n) == 1) ? 0x0024 : 0x0020)
+-
+-#define REG_INT_ENABLE(_n) \
+- (((_n) == 4) ? 0x0750 : \
+- ((_n) == 3) ? 0x0744 : \
+- ((_n) == 2) ? 0x0740 : \
+- ((_n) == 1) ? 0x002c : 0x0028)
+-
+-/* QDMA_CSR_INT_ENABLE1 */
+-#define RX15_COHERENT_INT_MASK BIT(31)
+-#define RX14_COHERENT_INT_MASK BIT(30)
+-#define RX13_COHERENT_INT_MASK BIT(29)
+-#define RX12_COHERENT_INT_MASK BIT(28)
+-#define RX11_COHERENT_INT_MASK BIT(27)
+-#define RX10_COHERENT_INT_MASK BIT(26)
+-#define RX9_COHERENT_INT_MASK BIT(25)
+-#define RX8_COHERENT_INT_MASK BIT(24)
+-#define RX7_COHERENT_INT_MASK BIT(23)
+-#define RX6_COHERENT_INT_MASK BIT(22)
+-#define RX5_COHERENT_INT_MASK BIT(21)
+-#define RX4_COHERENT_INT_MASK BIT(20)
+-#define RX3_COHERENT_INT_MASK BIT(19)
+-#define RX2_COHERENT_INT_MASK BIT(18)
+-#define RX1_COHERENT_INT_MASK BIT(17)
+-#define RX0_COHERENT_INT_MASK BIT(16)
+-#define TX7_COHERENT_INT_MASK BIT(15)
+-#define TX6_COHERENT_INT_MASK BIT(14)
+-#define TX5_COHERENT_INT_MASK BIT(13)
+-#define TX4_COHERENT_INT_MASK BIT(12)
+-#define TX3_COHERENT_INT_MASK BIT(11)
+-#define TX2_COHERENT_INT_MASK BIT(10)
+-#define TX1_COHERENT_INT_MASK BIT(9)
+-#define TX0_COHERENT_INT_MASK BIT(8)
+-#define CNT_OVER_FLOW_INT_MASK BIT(7)
+-#define IRQ1_FULL_INT_MASK BIT(5)
+-#define IRQ1_INT_MASK BIT(4)
+-#define HWFWD_DSCP_LOW_INT_MASK BIT(3)
+-#define HWFWD_DSCP_EMPTY_INT_MASK BIT(2)
+-#define IRQ0_FULL_INT_MASK BIT(1)
+-#define IRQ0_INT_MASK BIT(0)
+-
+-#define TX_DONE_INT_MASK(_n) \
+- ((_n) ? IRQ1_INT_MASK | IRQ1_FULL_INT_MASK \
+- : IRQ0_INT_MASK | IRQ0_FULL_INT_MASK)
+-
+-#define INT_TX_MASK \
+- (IRQ1_INT_MASK | IRQ1_FULL_INT_MASK | \
+- IRQ0_INT_MASK | IRQ0_FULL_INT_MASK)
+-
+-#define INT_IDX0_MASK \
+- (TX0_COHERENT_INT_MASK | TX1_COHERENT_INT_MASK | \
+- TX2_COHERENT_INT_MASK | TX3_COHERENT_INT_MASK | \
+- TX4_COHERENT_INT_MASK | TX5_COHERENT_INT_MASK | \
+- TX6_COHERENT_INT_MASK | TX7_COHERENT_INT_MASK | \
+- RX0_COHERENT_INT_MASK | RX1_COHERENT_INT_MASK | \
+- RX2_COHERENT_INT_MASK | RX3_COHERENT_INT_MASK | \
+- RX4_COHERENT_INT_MASK | RX7_COHERENT_INT_MASK | \
+- RX8_COHERENT_INT_MASK | RX9_COHERENT_INT_MASK | \
+- RX15_COHERENT_INT_MASK | INT_TX_MASK)
+-
+-/* QDMA_CSR_INT_ENABLE2 */
+-#define RX15_NO_CPU_DSCP_INT_MASK BIT(31)
+-#define RX14_NO_CPU_DSCP_INT_MASK BIT(30)
+-#define RX13_NO_CPU_DSCP_INT_MASK BIT(29)
+-#define RX12_NO_CPU_DSCP_INT_MASK BIT(28)
+-#define RX11_NO_CPU_DSCP_INT_MASK BIT(27)
+-#define RX10_NO_CPU_DSCP_INT_MASK BIT(26)
+-#define RX9_NO_CPU_DSCP_INT_MASK BIT(25)
+-#define RX8_NO_CPU_DSCP_INT_MASK BIT(24)
+-#define RX7_NO_CPU_DSCP_INT_MASK BIT(23)
+-#define RX6_NO_CPU_DSCP_INT_MASK BIT(22)
+-#define RX5_NO_CPU_DSCP_INT_MASK BIT(21)
+-#define RX4_NO_CPU_DSCP_INT_MASK BIT(20)
+-#define RX3_NO_CPU_DSCP_INT_MASK BIT(19)
+-#define RX2_NO_CPU_DSCP_INT_MASK BIT(18)
+-#define RX1_NO_CPU_DSCP_INT_MASK BIT(17)
+-#define RX0_NO_CPU_DSCP_INT_MASK BIT(16)
+-#define RX15_DONE_INT_MASK BIT(15)
+-#define RX14_DONE_INT_MASK BIT(14)
+-#define RX13_DONE_INT_MASK BIT(13)
+-#define RX12_DONE_INT_MASK BIT(12)
+-#define RX11_DONE_INT_MASK BIT(11)
+-#define RX10_DONE_INT_MASK BIT(10)
+-#define RX9_DONE_INT_MASK BIT(9)
+-#define RX8_DONE_INT_MASK BIT(8)
+-#define RX7_DONE_INT_MASK BIT(7)
+-#define RX6_DONE_INT_MASK BIT(6)
+-#define RX5_DONE_INT_MASK BIT(5)
+-#define RX4_DONE_INT_MASK BIT(4)
+-#define RX3_DONE_INT_MASK BIT(3)
+-#define RX2_DONE_INT_MASK BIT(2)
+-#define RX1_DONE_INT_MASK BIT(1)
+-#define RX0_DONE_INT_MASK BIT(0)
+-
+-#define RX_DONE_INT_MASK \
+- (RX0_DONE_INT_MASK | RX1_DONE_INT_MASK | \
+- RX2_DONE_INT_MASK | RX3_DONE_INT_MASK | \
+- RX4_DONE_INT_MASK | RX7_DONE_INT_MASK | \
+- RX8_DONE_INT_MASK | RX9_DONE_INT_MASK | \
+- RX15_DONE_INT_MASK)
+-#define INT_IDX1_MASK \
+- (RX_DONE_INT_MASK | \
+- RX0_NO_CPU_DSCP_INT_MASK | RX1_NO_CPU_DSCP_INT_MASK | \
+- RX2_NO_CPU_DSCP_INT_MASK | RX3_NO_CPU_DSCP_INT_MASK | \
+- RX4_NO_CPU_DSCP_INT_MASK | RX7_NO_CPU_DSCP_INT_MASK | \
+- RX8_NO_CPU_DSCP_INT_MASK | RX9_NO_CPU_DSCP_INT_MASK | \
+- RX15_NO_CPU_DSCP_INT_MASK)
+-
+-/* QDMA_CSR_INT_ENABLE5 */
+-#define TX31_COHERENT_INT_MASK BIT(31)
+-#define TX30_COHERENT_INT_MASK BIT(30)
+-#define TX29_COHERENT_INT_MASK BIT(29)
+-#define TX28_COHERENT_INT_MASK BIT(28)
+-#define TX27_COHERENT_INT_MASK BIT(27)
+-#define TX26_COHERENT_INT_MASK BIT(26)
+-#define TX25_COHERENT_INT_MASK BIT(25)
+-#define TX24_COHERENT_INT_MASK BIT(24)
+-#define TX23_COHERENT_INT_MASK BIT(23)
+-#define TX22_COHERENT_INT_MASK BIT(22)
+-#define TX21_COHERENT_INT_MASK BIT(21)
+-#define TX20_COHERENT_INT_MASK BIT(20)
+-#define TX19_COHERENT_INT_MASK BIT(19)
+-#define TX18_COHERENT_INT_MASK BIT(18)
+-#define TX17_COHERENT_INT_MASK BIT(17)
+-#define TX16_COHERENT_INT_MASK BIT(16)
+-#define TX15_COHERENT_INT_MASK BIT(15)
+-#define TX14_COHERENT_INT_MASK BIT(14)
+-#define TX13_COHERENT_INT_MASK BIT(13)
+-#define TX12_COHERENT_INT_MASK BIT(12)
+-#define TX11_COHERENT_INT_MASK BIT(11)
+-#define TX10_COHERENT_INT_MASK BIT(10)
+-#define TX9_COHERENT_INT_MASK BIT(9)
+-#define TX8_COHERENT_INT_MASK BIT(8)
+-
+-#define INT_IDX4_MASK \
+- (TX8_COHERENT_INT_MASK | TX9_COHERENT_INT_MASK | \
+- TX10_COHERENT_INT_MASK | TX11_COHERENT_INT_MASK | \
+- TX12_COHERENT_INT_MASK | TX13_COHERENT_INT_MASK | \
+- TX14_COHERENT_INT_MASK | TX15_COHERENT_INT_MASK | \
+- TX16_COHERENT_INT_MASK | TX17_COHERENT_INT_MASK | \
+- TX18_COHERENT_INT_MASK | TX19_COHERENT_INT_MASK | \
+- TX20_COHERENT_INT_MASK | TX21_COHERENT_INT_MASK | \
+- TX22_COHERENT_INT_MASK | TX23_COHERENT_INT_MASK | \
+- TX24_COHERENT_INT_MASK | TX25_COHERENT_INT_MASK | \
+- TX26_COHERENT_INT_MASK | TX27_COHERENT_INT_MASK | \
+- TX28_COHERENT_INT_MASK | TX29_COHERENT_INT_MASK | \
+- TX30_COHERENT_INT_MASK | TX31_COHERENT_INT_MASK)
+-
+-#define REG_TX_IRQ_BASE(_n) ((_n) ? 0x0048 : 0x0050)
+-
+-#define REG_TX_IRQ_CFG(_n) ((_n) ? 0x004c : 0x0054)
+-#define TX_IRQ_THR_MASK GENMASK(27, 16)
+-#define TX_IRQ_DEPTH_MASK GENMASK(11, 0)
+-
+-#define REG_IRQ_CLEAR_LEN(_n) ((_n) ? 0x0064 : 0x0058)
+-#define IRQ_CLEAR_LEN_MASK GENMASK(7, 0)
+-
+-#define REG_IRQ_STATUS(_n) ((_n) ? 0x0068 : 0x005c)
+-#define IRQ_ENTRY_LEN_MASK GENMASK(27, 16)
+-#define IRQ_HEAD_IDX_MASK GENMASK(11, 0)
+-
+-#define REG_TX_RING_BASE(_n) \
+- (((_n) < 8) ? 0x0100 + ((_n) << 5) : 0x0b00 + (((_n) - 8) << 5))
+-
+-#define REG_TX_RING_BLOCKING(_n) \
+- (((_n) < 8) ? 0x0104 + ((_n) << 5) : 0x0b04 + (((_n) - 8) << 5))
+-
+-#define TX_RING_IRQ_BLOCKING_MAP_MASK BIT(6)
+-#define TX_RING_IRQ_BLOCKING_CFG_MASK BIT(4)
+-#define TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK BIT(2)
+-#define TX_RING_IRQ_BLOCKING_MAX_TH_TXRING_EN_MASK BIT(1)
+-#define TX_RING_IRQ_BLOCKING_MIN_TH_TXRING_EN_MASK BIT(0)
+-
+-#define REG_TX_CPU_IDX(_n) \
+- (((_n) < 8) ? 0x0108 + ((_n) << 5) : 0x0b08 + (((_n) - 8) << 5))
+-
+-#define TX_RING_CPU_IDX_MASK GENMASK(15, 0)
+-
+-#define REG_TX_DMA_IDX(_n) \
+- (((_n) < 8) ? 0x010c + ((_n) << 5) : 0x0b0c + (((_n) - 8) << 5))
+-
+-#define TX_RING_DMA_IDX_MASK GENMASK(15, 0)
+-
+-#define IRQ_RING_IDX_MASK GENMASK(20, 16)
+-#define IRQ_DESC_IDX_MASK GENMASK(15, 0)
+-
+-#define REG_RX_RING_BASE(_n) \
+- (((_n) < 16) ? 0x0200 + ((_n) << 5) : 0x0e00 + (((_n) - 16) << 5))
+-
+-#define REG_RX_RING_SIZE(_n) \
+- (((_n) < 16) ? 0x0204 + ((_n) << 5) : 0x0e04 + (((_n) - 16) << 5))
+-
+-#define RX_RING_THR_MASK GENMASK(31, 16)
+-#define RX_RING_SIZE_MASK GENMASK(15, 0)
+-
+-#define REG_RX_CPU_IDX(_n) \
+- (((_n) < 16) ? 0x0208 + ((_n) << 5) : 0x0e08 + (((_n) - 16) << 5))
+-
+-#define RX_RING_CPU_IDX_MASK GENMASK(15, 0)
+-
+-#define REG_RX_DMA_IDX(_n) \
+- (((_n) < 16) ? 0x020c + ((_n) << 5) : 0x0e0c + (((_n) - 16) << 5))
+-
+-#define REG_RX_DELAY_INT_IDX(_n) \
+- (((_n) < 16) ? 0x0210 + ((_n) << 5) : 0x0e10 + (((_n) - 16) << 5))
+-
+-#define RX_DELAY_INT_MASK GENMASK(15, 0)
+-
+-#define RX_RING_DMA_IDX_MASK GENMASK(15, 0)
+-
+-#define REG_INGRESS_TRTCM_CFG 0x0070
+-#define INGRESS_TRTCM_EN_MASK BIT(31)
+-#define INGRESS_TRTCM_MODE_MASK BIT(30)
+-#define INGRESS_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
+-#define INGRESS_FAST_TICK_MASK GENMASK(15, 0)
+-
+-#define REG_QUEUE_CLOSE_CFG(_n) (0x00a0 + ((_n) & 0xfc))
+-#define TXQ_DISABLE_CHAN_QUEUE_MASK(_n, _m) BIT((_m) + (((_n) & 0x3) << 3))
+-
+-#define REG_TXQ_DIS_CFG_BASE(_n) ((_n) ? 0x20a0 : 0x00a0)
+-#define REG_TXQ_DIS_CFG(_n, _m) (REG_TXQ_DIS_CFG_BASE((_n)) + (_m) << 2)
+-
+-#define REG_CNTR_CFG(_n) (0x0400 + ((_n) << 3))
+-#define CNTR_EN_MASK BIT(31)
+-#define CNTR_ALL_CHAN_EN_MASK BIT(30)
+-#define CNTR_ALL_QUEUE_EN_MASK BIT(29)
+-#define CNTR_ALL_DSCP_RING_EN_MASK BIT(28)
+-#define CNTR_SRC_MASK GENMASK(27, 24)
+-#define CNTR_DSCP_RING_MASK GENMASK(20, 16)
+-#define CNTR_CHAN_MASK GENMASK(7, 3)
+-#define CNTR_QUEUE_MASK GENMASK(2, 0)
+-
+-#define REG_CNTR_VAL(_n) (0x0404 + ((_n) << 3))
+-
+-#define REG_LMGR_INIT_CFG 0x1000
+-#define LMGR_INIT_START BIT(31)
+-#define LMGR_SRAM_MODE_MASK BIT(30)
+-#define HW_FWD_PKTSIZE_OVERHEAD_MASK GENMASK(27, 20)
+-#define HW_FWD_DESC_NUM_MASK GENMASK(16, 0)
+-
+-#define REG_FWD_DSCP_LOW_THR 0x1004
+-#define FWD_DSCP_LOW_THR_MASK GENMASK(17, 0)
+-
+-#define REG_EGRESS_RATE_METER_CFG 0x100c
+-#define EGRESS_RATE_METER_EN_MASK BIT(29)
+-#define EGRESS_RATE_METER_EQ_RATE_EN_MASK BIT(17)
+-#define EGRESS_RATE_METER_WINDOW_SZ_MASK GENMASK(16, 12)
+-#define EGRESS_RATE_METER_TIMESLICE_MASK GENMASK(10, 0)
+-
+-#define REG_EGRESS_TRTCM_CFG 0x1010
+-#define EGRESS_TRTCM_EN_MASK BIT(31)
+-#define EGRESS_TRTCM_MODE_MASK BIT(30)
+-#define EGRESS_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
+-#define EGRESS_FAST_TICK_MASK GENMASK(15, 0)
+-
+-#define REG_TXWRR_MODE_CFG 0x1020
+-#define TWRR_WEIGHT_SCALE_MASK BIT(31)
+-#define TWRR_WEIGHT_BASE_MASK BIT(3)
+-
+-#define REG_TXWRR_WEIGHT_CFG 0x1024
+-#define TWRR_RW_CMD_MASK BIT(31)
+-#define TWRR_RW_CMD_DONE BIT(30)
+-#define TWRR_CHAN_IDX_MASK GENMASK(23, 19)
+-#define TWRR_QUEUE_IDX_MASK GENMASK(18, 16)
+-#define TWRR_VALUE_MASK GENMASK(15, 0)
+-
+-#define REG_PSE_BUF_USAGE_CFG 0x1028
+-#define PSE_BUF_ESTIMATE_EN_MASK BIT(29)
+-
+-#define REG_CHAN_QOS_MODE(_n) (0x1040 + ((_n) << 2))
+-#define CHAN_QOS_MODE_MASK(_n) GENMASK(2 + ((_n) << 2), (_n) << 2)
+-
+-#define REG_GLB_TRTCM_CFG 0x1080
+-#define GLB_TRTCM_EN_MASK BIT(31)
+-#define GLB_TRTCM_MODE_MASK BIT(30)
+-#define GLB_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
+-#define GLB_FAST_TICK_MASK GENMASK(15, 0)
+-
+-#define REG_TXQ_CNGST_CFG 0x10a0
+-#define TXQ_CNGST_DROP_EN BIT(31)
+-#define TXQ_CNGST_DEI_DROP_EN BIT(30)
+-
+-#define REG_SLA_TRTCM_CFG 0x1150
+-#define SLA_TRTCM_EN_MASK BIT(31)
+-#define SLA_TRTCM_MODE_MASK BIT(30)
+-#define SLA_SLOW_TICK_RATIO_MASK GENMASK(29, 16)
+-#define SLA_FAST_TICK_MASK GENMASK(15, 0)
+-
+-/* CTRL */
+-#define QDMA_DESC_DONE_MASK BIT(31)
+-#define QDMA_DESC_DROP_MASK BIT(30) /* tx: drop - rx: overflow */
+-#define QDMA_DESC_MORE_MASK BIT(29) /* more SG elements */
+-#define QDMA_DESC_DEI_MASK BIT(25)
+-#define QDMA_DESC_NO_DROP_MASK BIT(24)
+-#define QDMA_DESC_LEN_MASK GENMASK(15, 0)
+-/* DATA */
+-#define QDMA_DESC_NEXT_ID_MASK GENMASK(15, 0)
+-/* TX MSG0 */
+-#define QDMA_ETH_TXMSG_MIC_IDX_MASK BIT(30)
+-#define QDMA_ETH_TXMSG_SP_TAG_MASK GENMASK(29, 14)
+-#define QDMA_ETH_TXMSG_ICO_MASK BIT(13)
+-#define QDMA_ETH_TXMSG_UCO_MASK BIT(12)
+-#define QDMA_ETH_TXMSG_TCO_MASK BIT(11)
+-#define QDMA_ETH_TXMSG_TSO_MASK BIT(10)
+-#define QDMA_ETH_TXMSG_FAST_MASK BIT(9)
+-#define QDMA_ETH_TXMSG_OAM_MASK BIT(8)
+-#define QDMA_ETH_TXMSG_CHAN_MASK GENMASK(7, 3)
+-#define QDMA_ETH_TXMSG_QUEUE_MASK GENMASK(2, 0)
+-/* TX MSG1 */
+-#define QDMA_ETH_TXMSG_NO_DROP BIT(31)
+-#define QDMA_ETH_TXMSG_METER_MASK GENMASK(30, 24) /* 0x7f no meters */
+-#define QDMA_ETH_TXMSG_FPORT_MASK GENMASK(23, 20)
+-#define QDMA_ETH_TXMSG_NBOQ_MASK GENMASK(19, 15)
+-#define QDMA_ETH_TXMSG_HWF_MASK BIT(14)
+-#define QDMA_ETH_TXMSG_HOP_MASK BIT(13)
+-#define QDMA_ETH_TXMSG_PTP_MASK BIT(12)
+-#define QDMA_ETH_TXMSG_ACNT_G1_MASK GENMASK(10, 6) /* 0x1f do not count */
+-#define QDMA_ETH_TXMSG_ACNT_G0_MASK GENMASK(5, 0) /* 0x3f do not count */
+-
+-/* RX MSG1 */
+-#define QDMA_ETH_RXMSG_DEI_MASK BIT(31)
+-#define QDMA_ETH_RXMSG_IP6_MASK BIT(30)
+-#define QDMA_ETH_RXMSG_IP4_MASK BIT(29)
+-#define QDMA_ETH_RXMSG_IP4F_MASK BIT(28)
+-#define QDMA_ETH_RXMSG_L4_VALID_MASK BIT(27)
+-#define QDMA_ETH_RXMSG_L4F_MASK BIT(26)
+-#define QDMA_ETH_RXMSG_SPORT_MASK GENMASK(25, 21)
+-#define QDMA_ETH_RXMSG_CRSN_MASK GENMASK(20, 16)
+-#define QDMA_ETH_RXMSG_PPE_ENTRY_MASK GENMASK(15, 0)
+-
+-struct airoha_qdma_desc {
+- __le32 rsv;
+- __le32 ctrl;
+- __le32 addr;
+- __le32 data;
+- __le32 msg0;
+- __le32 msg1;
+- __le32 msg2;
+- __le32 msg3;
+-};
+-
+-/* CTRL0 */
+-#define QDMA_FWD_DESC_CTX_MASK BIT(31)
+-#define QDMA_FWD_DESC_RING_MASK GENMASK(30, 28)
+-#define QDMA_FWD_DESC_IDX_MASK GENMASK(27, 16)
+-#define QDMA_FWD_DESC_LEN_MASK GENMASK(15, 0)
+-/* CTRL1 */
+-#define QDMA_FWD_DESC_FIRST_IDX_MASK GENMASK(15, 0)
+-/* CTRL2 */
+-#define QDMA_FWD_DESC_MORE_PKT_NUM_MASK GENMASK(2, 0)
+-
+-struct airoha_qdma_fwd_desc {
+- __le32 addr;
+- __le32 ctrl0;
+- __le32 ctrl1;
+- __le32 ctrl2;
+- __le32 msg0;
+- __le32 msg1;
+- __le32 rsv0;
+- __le32 rsv1;
+-};
+-
+-enum {
+- QDMA_INT_REG_IDX0,
+- QDMA_INT_REG_IDX1,
+- QDMA_INT_REG_IDX2,
+- QDMA_INT_REG_IDX3,
+- QDMA_INT_REG_IDX4,
+- QDMA_INT_REG_MAX
+-};
+-
+-enum {
+- XSI_PCIE0_PORT,
+- XSI_PCIE1_PORT,
+- XSI_USB_PORT,
+- XSI_AE_PORT,
+- XSI_ETH_PORT,
+-};
+-
+-enum {
+- XSI_PCIE0_VIP_PORT_MASK = BIT(22),
+- XSI_PCIE1_VIP_PORT_MASK = BIT(23),
+- XSI_USB_VIP_PORT_MASK = BIT(25),
+- XSI_ETH_VIP_PORT_MASK = BIT(24),
+-};
+-
+-enum {
+- DEV_STATE_INITIALIZED,
+-};
+-
+-enum {
+- CDM_CRSN_QSEL_Q1 = 1,
+- CDM_CRSN_QSEL_Q5 = 5,
+- CDM_CRSN_QSEL_Q6 = 6,
+- CDM_CRSN_QSEL_Q15 = 15,
+-};
+-
+-enum {
+- CRSN_08 = 0x8,
+- CRSN_21 = 0x15, /* KA */
+- CRSN_22 = 0x16, /* hit bind and force route to CPU */
+- CRSN_24 = 0x18,
+- CRSN_25 = 0x19,
+-};
+-
+-enum {
+- FE_PSE_PORT_CDM1,
+- FE_PSE_PORT_GDM1,
+- FE_PSE_PORT_GDM2,
+- FE_PSE_PORT_GDM3,
+- FE_PSE_PORT_PPE1,
+- FE_PSE_PORT_CDM2,
+- FE_PSE_PORT_CDM3,
+- FE_PSE_PORT_CDM4,
+- FE_PSE_PORT_PPE2,
+- FE_PSE_PORT_GDM4,
+- FE_PSE_PORT_CDM5,
+- FE_PSE_PORT_DROP = 0xf,
+-};
+-
+-enum tx_sched_mode {
+- TC_SCH_WRR8,
+- TC_SCH_SP,
+- TC_SCH_WRR7,
+- TC_SCH_WRR6,
+- TC_SCH_WRR5,
+- TC_SCH_WRR4,
+- TC_SCH_WRR3,
+- TC_SCH_WRR2,
+-};
+-
+-struct airoha_queue_entry {
+- union {
+- void *buf;
+- struct sk_buff *skb;
+- };
+- dma_addr_t dma_addr;
+- u16 dma_len;
+-};
+-
+-struct airoha_queue {
+- struct airoha_qdma *qdma;
+-
+- /* protect concurrent queue accesses */
+- spinlock_t lock;
+- struct airoha_queue_entry *entry;
+- struct airoha_qdma_desc *desc;
+- u16 head;
+- u16 tail;
+-
+- int queued;
+- int ndesc;
+- int free_thr;
+- int buf_size;
+-
+- struct napi_struct napi;
+- struct page_pool *page_pool;
+-};
+-
+-struct airoha_tx_irq_queue {
+- struct airoha_qdma *qdma;
+-
+- struct napi_struct napi;
+-
+- int size;
+- u32 *q;
+-};
+-
+-struct airoha_hw_stats {
+- /* protect concurrent hw_stats accesses */
+- spinlock_t lock;
+- struct u64_stats_sync syncp;
+-
+- /* get_stats64 */
+- u64 rx_ok_pkts;
+- u64 tx_ok_pkts;
+- u64 rx_ok_bytes;
+- u64 tx_ok_bytes;
+- u64 rx_multicast;
+- u64 rx_errors;
+- u64 rx_drops;
+- u64 tx_drops;
+- u64 rx_crc_error;
+- u64 rx_over_errors;
+- /* ethtool stats */
+- u64 tx_broadcast;
+- u64 tx_multicast;
+- u64 tx_len[7];
+- u64 rx_broadcast;
+- u64 rx_fragment;
+- u64 rx_jabber;
+- u64 rx_len[7];
+-};
+-
+-struct airoha_qdma {
+- struct airoha_eth *eth;
+- void __iomem *regs;
+-
+- /* protect concurrent irqmask accesses */
+- spinlock_t irq_lock;
+- u32 irqmask[QDMA_INT_REG_MAX];
+- int irq;
+-
+- struct airoha_tx_irq_queue q_tx_irq[AIROHA_NUM_TX_IRQ];
+-
+- struct airoha_queue q_tx[AIROHA_NUM_TX_RING];
+- struct airoha_queue q_rx[AIROHA_NUM_RX_RING];
+-
+- /* descriptor and packet buffers for qdma hw forward */
+- struct {
+- void *desc;
+- void *q;
+- } hfwd;
+-};
+-
+-struct airoha_gdm_port {
+- struct airoha_qdma *qdma;
+- struct net_device *dev;
+- int id;
+-
+- struct airoha_hw_stats stats;
+-
+- /* qos stats counters */
+- u64 cpu_tx_packets;
+- u64 fwd_tx_packets;
+-};
+-
+-struct airoha_eth {
+- struct device *dev;
+-
+- unsigned long state;
+- void __iomem *fe_regs;
+-
+- struct reset_control_bulk_data rsts[AIROHA_MAX_NUM_RSTS];
+- struct reset_control_bulk_data xsi_rsts[AIROHA_MAX_NUM_XSI_RSTS];
+-
+- struct net_device *napi_dev;
+-
+- struct airoha_qdma qdma[AIROHA_MAX_NUM_QDMA];
+- struct airoha_gdm_port *ports[AIROHA_MAX_NUM_GDM_PORTS];
+-};
+-
+-static u32 airoha_rr(void __iomem *base, u32 offset)
+-{
+- return readl(base + offset);
+-}
+-
+-static void airoha_wr(void __iomem *base, u32 offset, u32 val)
+-{
+- writel(val, base + offset);
+-}
+-
+-static u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val)
+-{
+- val |= (airoha_rr(base, offset) & ~mask);
+- airoha_wr(base, offset, val);
+-
+- return val;
+-}
+-
+-#define airoha_fe_rr(eth, offset) \
+- airoha_rr((eth)->fe_regs, (offset))
+-#define airoha_fe_wr(eth, offset, val) \
+- airoha_wr((eth)->fe_regs, (offset), (val))
+-#define airoha_fe_rmw(eth, offset, mask, val) \
+- airoha_rmw((eth)->fe_regs, (offset), (mask), (val))
+-#define airoha_fe_set(eth, offset, val) \
+- airoha_rmw((eth)->fe_regs, (offset), 0, (val))
+-#define airoha_fe_clear(eth, offset, val) \
+- airoha_rmw((eth)->fe_regs, (offset), (val), 0)
+-
+-#define airoha_qdma_rr(qdma, offset) \
+- airoha_rr((qdma)->regs, (offset))
+-#define airoha_qdma_wr(qdma, offset, val) \
+- airoha_wr((qdma)->regs, (offset), (val))
+-#define airoha_qdma_rmw(qdma, offset, mask, val) \
+- airoha_rmw((qdma)->regs, (offset), (mask), (val))
+-#define airoha_qdma_set(qdma, offset, val) \
+- airoha_rmw((qdma)->regs, (offset), 0, (val))
+-#define airoha_qdma_clear(qdma, offset, val) \
+- airoha_rmw((qdma)->regs, (offset), (val), 0)
+-
+-static void airoha_qdma_set_irqmask(struct airoha_qdma *qdma, int index,
+- u32 clear, u32 set)
+-{
+- unsigned long flags;
+-
+- if (WARN_ON_ONCE(index >= ARRAY_SIZE(qdma->irqmask)))
+- return;
+-
+- spin_lock_irqsave(&qdma->irq_lock, flags);
+-
+- qdma->irqmask[index] &= ~clear;
+- qdma->irqmask[index] |= set;
+- airoha_qdma_wr(qdma, REG_INT_ENABLE(index), qdma->irqmask[index]);
+- /* Read irq_enable register in order to guarantee the update above
+- * completes in the spinlock critical section.
+- */
+- airoha_qdma_rr(qdma, REG_INT_ENABLE(index));
+-
+- spin_unlock_irqrestore(&qdma->irq_lock, flags);
+-}
+-
+-static void airoha_qdma_irq_enable(struct airoha_qdma *qdma, int index,
+- u32 mask)
+-{
+- airoha_qdma_set_irqmask(qdma, index, 0, mask);
+-}
+-
+-static void airoha_qdma_irq_disable(struct airoha_qdma *qdma, int index,
+- u32 mask)
+-{
+- airoha_qdma_set_irqmask(qdma, index, mask, 0);
+-}
+-
+-static bool airhoa_is_lan_gdm_port(struct airoha_gdm_port *port)
+-{
+- /* GDM1 port on EN7581 SoC is connected to the lan dsa switch.
+- * GDM{2,3,4} can be used as wan port connected to an external
+- * phy module.
+- */
+- return port->id == 1;
+-}
+-
+-static void airoha_set_macaddr(struct airoha_gdm_port *port, const u8 *addr)
+-{
+- struct airoha_eth *eth = port->qdma->eth;
+- u32 val, reg;
+-
+- reg = airhoa_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H
+- : REG_FE_WAN_MAC_H;
+- val = (addr[0] << 16) | (addr[1] << 8) | addr[2];
+- airoha_fe_wr(eth, reg, val);
+-
+- val = (addr[3] << 16) | (addr[4] << 8) | addr[5];
+- airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), val);
+- airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), val);
+-}
+-
+-static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr,
+- u32 val)
+-{
+- airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK,
+- FIELD_PREP(GDM_OCFQ_MASK, val));
+- airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK,
+- FIELD_PREP(GDM_MCFQ_MASK, val));
+- airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK,
+- FIELD_PREP(GDM_BCFQ_MASK, val));
+- airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK,
+- FIELD_PREP(GDM_UCFQ_MASK, val));
+-}
+-
+-static int airoha_set_gdm_port(struct airoha_eth *eth, int port, bool enable)
+-{
+- u32 val = enable ? FE_PSE_PORT_PPE1 : FE_PSE_PORT_DROP;
+- u32 vip_port, cfg_addr;
+-
+- switch (port) {
+- case XSI_PCIE0_PORT:
+- vip_port = XSI_PCIE0_VIP_PORT_MASK;
+- cfg_addr = REG_GDM_FWD_CFG(3);
+- break;
+- case XSI_PCIE1_PORT:
+- vip_port = XSI_PCIE1_VIP_PORT_MASK;
+- cfg_addr = REG_GDM_FWD_CFG(3);
+- break;
+- case XSI_USB_PORT:
+- vip_port = XSI_USB_VIP_PORT_MASK;
+- cfg_addr = REG_GDM_FWD_CFG(4);
+- break;
+- case XSI_ETH_PORT:
+- vip_port = XSI_ETH_VIP_PORT_MASK;
+- cfg_addr = REG_GDM_FWD_CFG(4);
+- break;
+- default:
+- return -EINVAL;
+- }
+-
+- if (enable) {
+- airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port);
+- airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port);
+- } else {
+- airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port);
+- airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port);
+- }
+-
+- airoha_set_gdm_port_fwd_cfg(eth, cfg_addr, val);
+-
+- return 0;
+-}
+-
+-static int airoha_set_gdm_ports(struct airoha_eth *eth, bool enable)
+-{
+- const int port_list[] = {
+- XSI_PCIE0_PORT,
+- XSI_PCIE1_PORT,
+- XSI_USB_PORT,
+- XSI_ETH_PORT
+- };
+- int i, err;
+-
+- for (i = 0; i < ARRAY_SIZE(port_list); i++) {
+- err = airoha_set_gdm_port(eth, port_list[i], enable);
+- if (err)
+- goto error;
+- }
+-
+- return 0;
+-
+-error:
+- for (i--; i >= 0; i--)
+- airoha_set_gdm_port(eth, port_list[i], false);
+-
+- return err;
+-}
+-
+-static void airoha_fe_maccr_init(struct airoha_eth *eth)
+-{
+- int p;
+-
+- for (p = 1; p <= ARRAY_SIZE(eth->ports); p++) {
+- airoha_fe_set(eth, REG_GDM_FWD_CFG(p),
+- GDM_TCP_CKSUM | GDM_UDP_CKSUM | GDM_IP4_CKSUM |
+- GDM_DROP_CRC_ERR);
+- airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(p),
+- FE_PSE_PORT_CDM1);
+- airoha_fe_rmw(eth, REG_GDM_LEN_CFG(p),
+- GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK,
+- FIELD_PREP(GDM_SHORT_LEN_MASK, 60) |
+- FIELD_PREP(GDM_LONG_LEN_MASK, 4004));
+- }
+-
+- airoha_fe_rmw(eth, REG_CDM1_VLAN_CTRL, CDM1_VLAN_MASK,
+- FIELD_PREP(CDM1_VLAN_MASK, 0x8100));
+-
+- airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD);
+-}
+-
+-static void airoha_fe_vip_setup(struct airoha_eth *eth)
+-{
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK);
+-
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(4),
+- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+- PATN_EN_MASK);
+-
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(6),
+- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+- PATN_EN_MASK);
+-
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(7),
+- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+- PATN_EN_MASK);
+-
+- /* BOOTP (0x43) */
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(8),
+- PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
+- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+-
+- /* BOOTP (0x44) */
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(9),
+- PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
+- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+-
+- /* ISAKMP */
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(10),
+- PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
+- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+-
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(11),
+- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+- PATN_EN_MASK);
+-
+- /* DHCPv6 */
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(12),
+- PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
+- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+-
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(19),
+- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+- PATN_EN_MASK);
+-
+- /* ETH->ETH_P_1905 (0x893a) */
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(20),
+- PATN_FCPU_EN_MASK | PATN_EN_MASK);
+-
+- airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP);
+- airoha_fe_wr(eth, REG_FE_VIP_EN(21),
+- PATN_FCPU_EN_MASK | PATN_EN_MASK);
+-}
+-
+-static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth,
+- u32 port, u32 queue)
+-{
+- u32 val;
+-
+- airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
+- PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK,
+- FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
+- FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue));
+- val = airoha_fe_rr(eth, REG_FE_PSE_QUEUE_CFG_VAL);
+-
+- return FIELD_GET(PSE_CFG_OQ_RSV_MASK, val);
+-}
+-
+-static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth,
+- u32 port, u32 queue, u32 val)
+-{
+- airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK,
+- FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val));
+- airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
+- PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK |
+- PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK,
+- FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
+- FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) |
+- PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK);
+-}
+-
+-static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth)
+-{
+- u32 val = airoha_fe_rr(eth, REG_FE_PSE_BUF_SET);
+-
+- return FIELD_GET(PSE_ALLRSV_MASK, val);
+-}
+-
+-static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth,
+- u32 port, u32 queue, u32 val)
+-{
+- u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue);
+- u32 tmp, all_rsv, fq_limit;
+-
+- airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val);
+-
+- /* modify all rsv */
+- all_rsv = airoha_fe_get_pse_all_rsv(eth);
+- all_rsv += (val - orig_val);
+- airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
+- FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
+-
+- /* modify hthd */
+- tmp = airoha_fe_rr(eth, PSE_FQ_CFG);
+- fq_limit = FIELD_GET(PSE_FQ_LIMIT_MASK, tmp);
+- tmp = fq_limit - all_rsv - 0x20;
+- airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
+- PSE_SHARE_USED_HTHD_MASK,
+- FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp));
+-
+- tmp = fq_limit - all_rsv - 0x100;
+- airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
+- PSE_SHARE_USED_MTHD_MASK,
+- FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp));
+- tmp = (3 * tmp) >> 2;
+- airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET,
+- PSE_SHARE_USED_LTHD_MASK,
+- FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp));
+-
+- return 0;
+-}
+-
+-static void airoha_fe_pse_ports_init(struct airoha_eth *eth)
+-{
+- const u32 pse_port_num_queues[] = {
+- [FE_PSE_PORT_CDM1] = 6,
+- [FE_PSE_PORT_GDM1] = 6,
+- [FE_PSE_PORT_GDM2] = 32,
+- [FE_PSE_PORT_GDM3] = 6,
+- [FE_PSE_PORT_PPE1] = 4,
+- [FE_PSE_PORT_CDM2] = 6,
+- [FE_PSE_PORT_CDM3] = 8,
+- [FE_PSE_PORT_CDM4] = 10,
+- [FE_PSE_PORT_PPE2] = 4,
+- [FE_PSE_PORT_GDM4] = 2,
+- [FE_PSE_PORT_CDM5] = 2,
+- };
+- u32 all_rsv;
+- int q;
+-
+- all_rsv = airoha_fe_get_pse_all_rsv(eth);
+- /* hw misses PPE2 oq rsv */
+- all_rsv += PSE_RSV_PAGES * pse_port_num_queues[FE_PSE_PORT_PPE2];
+- airoha_fe_set(eth, REG_FE_PSE_BUF_SET, all_rsv);
+-
+- /* CMD1 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q,
+- PSE_QUEUE_RSV_PAGES);
+- /* GMD1 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q,
+- PSE_QUEUE_RSV_PAGES);
+- /* GMD2 */
+- for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0);
+- /* GMD3 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q,
+- PSE_QUEUE_RSV_PAGES);
+- /* PPE1 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) {
+- if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q,
+- PSE_QUEUE_RSV_PAGES);
+- else
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0);
+- }
+- /* CDM2 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q,
+- PSE_QUEUE_RSV_PAGES);
+- /* CDM3 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0);
+- /* CDM4 */
+- for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q,
+- PSE_QUEUE_RSV_PAGES);
+- /* PPE2 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) {
+- if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, q,
+- PSE_QUEUE_RSV_PAGES);
+- else
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2, q, 0);
+- }
+- /* GMD4 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q,
+- PSE_QUEUE_RSV_PAGES);
+- /* CDM5 */
+- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++)
+- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q,
+- PSE_QUEUE_RSV_PAGES);
+-}
+-
+-static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth)
+-{
+- int i;
+-
+- for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) {
+- int err, j;
+- u32 val;
+-
+- airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
+-
+- val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
+- MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK;
+- airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
+- err = read_poll_timeout(airoha_fe_rr, val,
+- val & MC_VLAN_CFG_CMD_DONE_MASK,
+- USEC_PER_MSEC, 5 * USEC_PER_MSEC,
+- false, eth, REG_MC_VLAN_CFG);
+- if (err)
+- return err;
+-
+- for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) {
+- airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
+-
+- val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
+- FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) |
+- MC_VLAN_CFG_RW_MASK;
+- airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
+- err = read_poll_timeout(airoha_fe_rr, val,
+- val & MC_VLAN_CFG_CMD_DONE_MASK,
+- USEC_PER_MSEC,
+- 5 * USEC_PER_MSEC, false, eth,
+- REG_MC_VLAN_CFG);
+- if (err)
+- return err;
+- }
+- }
+-
+- return 0;
+-}
+-
+-static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth)
+-{
+- /* CDM1_CRSN_QSEL */
+- airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_22 >> 2),
+- CDM1_CRSN_QSEL_REASON_MASK(CRSN_22),
+- FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_22),
+- CDM_CRSN_QSEL_Q1));
+- airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_08 >> 2),
+- CDM1_CRSN_QSEL_REASON_MASK(CRSN_08),
+- FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_08),
+- CDM_CRSN_QSEL_Q1));
+- airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_21 >> 2),
+- CDM1_CRSN_QSEL_REASON_MASK(CRSN_21),
+- FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_21),
+- CDM_CRSN_QSEL_Q1));
+- airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_24 >> 2),
+- CDM1_CRSN_QSEL_REASON_MASK(CRSN_24),
+- FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_24),
+- CDM_CRSN_QSEL_Q6));
+- airoha_fe_rmw(eth, REG_CDM1_CRSN_QSEL(CRSN_25 >> 2),
+- CDM1_CRSN_QSEL_REASON_MASK(CRSN_25),
+- FIELD_PREP(CDM1_CRSN_QSEL_REASON_MASK(CRSN_25),
+- CDM_CRSN_QSEL_Q1));
+- /* CDM2_CRSN_QSEL */
+- airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_08 >> 2),
+- CDM2_CRSN_QSEL_REASON_MASK(CRSN_08),
+- FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_08),
+- CDM_CRSN_QSEL_Q1));
+- airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_21 >> 2),
+- CDM2_CRSN_QSEL_REASON_MASK(CRSN_21),
+- FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_21),
+- CDM_CRSN_QSEL_Q1));
+- airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_22 >> 2),
+- CDM2_CRSN_QSEL_REASON_MASK(CRSN_22),
+- FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_22),
+- CDM_CRSN_QSEL_Q1));
+- airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_24 >> 2),
+- CDM2_CRSN_QSEL_REASON_MASK(CRSN_24),
+- FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_24),
+- CDM_CRSN_QSEL_Q6));
+- airoha_fe_rmw(eth, REG_CDM2_CRSN_QSEL(CRSN_25 >> 2),
+- CDM2_CRSN_QSEL_REASON_MASK(CRSN_25),
+- FIELD_PREP(CDM2_CRSN_QSEL_REASON_MASK(CRSN_25),
+- CDM_CRSN_QSEL_Q1));
+-}
+-
+-static int airoha_fe_init(struct airoha_eth *eth)
+-{
+- airoha_fe_maccr_init(eth);
+-
+- /* PSE IQ reserve */
+- airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK,
+- FIELD_PREP(PSE_IQ_RES1_P2_MASK, 0x10));
+- airoha_fe_rmw(eth, REG_PSE_IQ_REV2,
+- PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK,
+- FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) |
+- FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34));
+-
+- /* enable FE copy engine for MC/KA/DPI */
+- airoha_fe_wr(eth, REG_FE_PCE_CFG,
+- PCE_DPI_EN_MASK | PCE_KA_EN_MASK | PCE_MC_EN_MASK);
+- /* set vip queue selection to ring 1 */
+- airoha_fe_rmw(eth, REG_CDM1_FWD_CFG, CDM1_VIP_QSEL_MASK,
+- FIELD_PREP(CDM1_VIP_QSEL_MASK, 0x4));
+- airoha_fe_rmw(eth, REG_CDM2_FWD_CFG, CDM2_VIP_QSEL_MASK,
+- FIELD_PREP(CDM2_VIP_QSEL_MASK, 0x4));
+- /* set GDM4 source interface offset to 8 */
+- airoha_fe_rmw(eth, REG_GDM4_SRC_PORT_SET,
+- GDM4_SPORT_OFF2_MASK |
+- GDM4_SPORT_OFF1_MASK |
+- GDM4_SPORT_OFF0_MASK,
+- FIELD_PREP(GDM4_SPORT_OFF2_MASK, 8) |
+- FIELD_PREP(GDM4_SPORT_OFF1_MASK, 8) |
+- FIELD_PREP(GDM4_SPORT_OFF0_MASK, 8));
+-
+- /* set PSE Page as 128B */
+- airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG,
+- FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK,
+- FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, 2) |
+- FE_DMA_GLO_PG_SZ_MASK);
+- airoha_fe_wr(eth, REG_FE_RST_GLO_CFG,
+- FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK |
+- FE_RST_GDM4_MBI_ARB_MASK);
+- usleep_range(1000, 2000);
+-
+- /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1
+- * connect other rings to PSE Port0 OQ-0
+- */
+- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4));
+- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28));
+- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4));
+- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28));
+-
+- airoha_fe_vip_setup(eth);
+- airoha_fe_pse_ports_init(eth);
+-
+- airoha_fe_set(eth, REG_GDM_MISC_CFG,
+- GDM2_RDM_ACK_WAIT_PREF_MASK |
+- GDM2_CHN_VLD_MODE_MASK);
+- airoha_fe_rmw(eth, REG_CDM2_FWD_CFG, CDM2_OAM_QSEL_MASK, 15);
+-
+- /* init fragment and assemble Force Port */
+- /* NPU Core-3, NPU Bridge Channel-3 */
+- airoha_fe_rmw(eth, REG_IP_FRAG_FP,
+- IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK,
+- FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) |
+- FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3));
+- /* QDMA LAN, RX Ring-22 */
+- airoha_fe_rmw(eth, REG_IP_FRAG_FP,
+- IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK,
+- FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) |
+- FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22));
+-
+- airoha_fe_set(eth, REG_GDM3_FWD_CFG, GDM3_PAD_EN_MASK);
+- airoha_fe_set(eth, REG_GDM4_FWD_CFG, GDM4_PAD_EN_MASK);
+-
+- airoha_fe_crsn_qsel_init(eth);
+-
+- airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK);
+- airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK);
+-
+- /* default aging mode for mbi unlock issue */
+- airoha_fe_rmw(eth, REG_GDM2_CHN_RLS,
+- MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK,
+- FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) |
+- FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3));
+-
+- /* disable IFC by default */
+- airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK);
+-
+- /* enable 1:N vlan action, init vlan table */
+- airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK);
+-
+- return airoha_fe_mc_vlan_clear(eth);
+-}
+-
+-static int airoha_qdma_fill_rx_queue(struct airoha_queue *q)
+-{
+- enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
+- struct airoha_qdma *qdma = q->qdma;
+- struct airoha_eth *eth = qdma->eth;
+- int qid = q - &qdma->q_rx[0];
+- int nframes = 0;
+-
+- while (q->queued < q->ndesc - 1) {
+- struct airoha_queue_entry *e = &q->entry[q->head];
+- struct airoha_qdma_desc *desc = &q->desc[q->head];
+- struct page *page;
+- int offset;
+- u32 val;
+-
+- page = page_pool_dev_alloc_frag(q->page_pool, &offset,
+- q->buf_size);
+- if (!page)
+- break;
+-
+- q->head = (q->head + 1) % q->ndesc;
+- q->queued++;
+- nframes++;
+-
+- e->buf = page_address(page) + offset;
+- e->dma_addr = page_pool_get_dma_addr(page) + offset;
+- e->dma_len = SKB_WITH_OVERHEAD(q->buf_size);
+-
+- dma_sync_single_for_device(eth->dev, e->dma_addr, e->dma_len,
+- dir);
+-
+- val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len);
+- WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
+- WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr));
+- val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head);
+- WRITE_ONCE(desc->data, cpu_to_le32(val));
+- WRITE_ONCE(desc->msg0, 0);
+- WRITE_ONCE(desc->msg1, 0);
+- WRITE_ONCE(desc->msg2, 0);
+- WRITE_ONCE(desc->msg3, 0);
+-
+- airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid),
+- RX_RING_CPU_IDX_MASK,
+- FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
+- }
+-
+- return nframes;
+-}
+-
+-static int airoha_qdma_get_gdm_port(struct airoha_eth *eth,
+- struct airoha_qdma_desc *desc)
+-{
+- u32 port, sport, msg1 = le32_to_cpu(desc->msg1);
+-
+- sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, msg1);
+- switch (sport) {
+- case 0x10 ... 0x13:
+- port = 0;
+- break;
+- case 0x2 ... 0x4:
+- port = sport - 1;
+- break;
+- default:
+- return -EINVAL;
+- }
+-
+- return port >= ARRAY_SIZE(eth->ports) ? -EINVAL : port;
+-}
+-
+-static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
+-{
+- enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
+- struct airoha_qdma *qdma = q->qdma;
+- struct airoha_eth *eth = qdma->eth;
+- int qid = q - &qdma->q_rx[0];
+- int done = 0;
+-
+- while (done < budget) {
+- struct airoha_queue_entry *e = &q->entry[q->tail];
+- struct airoha_qdma_desc *desc = &q->desc[q->tail];
+- dma_addr_t dma_addr = le32_to_cpu(desc->addr);
+- u32 desc_ctrl = le32_to_cpu(desc->ctrl);
+- struct sk_buff *skb;
+- int len, p;
+-
+- if (!(desc_ctrl & QDMA_DESC_DONE_MASK))
+- break;
+-
+- if (!dma_addr)
+- break;
+-
+- len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
+- if (!len)
+- break;
+-
+- q->tail = (q->tail + 1) % q->ndesc;
+- q->queued--;
+-
+- dma_sync_single_for_cpu(eth->dev, dma_addr,
+- SKB_WITH_OVERHEAD(q->buf_size), dir);
+-
+- p = airoha_qdma_get_gdm_port(eth, desc);
+- if (p < 0 || !eth->ports[p]) {
+- page_pool_put_full_page(q->page_pool,
+- virt_to_head_page(e->buf),
+- true);
+- continue;
+- }
+-
+- skb = napi_build_skb(e->buf, q->buf_size);
+- if (!skb) {
+- page_pool_put_full_page(q->page_pool,
+- virt_to_head_page(e->buf),
+- true);
+- break;
+- }
+-
+- skb_reserve(skb, 2);
+- __skb_put(skb, len);
+- skb_mark_for_recycle(skb);
+- skb->dev = eth->ports[p]->dev;
+- skb->protocol = eth_type_trans(skb, skb->dev);
+- skb->ip_summed = CHECKSUM_UNNECESSARY;
+- skb_record_rx_queue(skb, qid);
+- napi_gro_receive(&q->napi, skb);
+-
+- done++;
+- }
+- airoha_qdma_fill_rx_queue(q);
+-
+- return done;
+-}
+-
+-static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
+-{
+- struct airoha_queue *q = container_of(napi, struct airoha_queue, napi);
+- int cur, done = 0;
+-
+- do {
+- cur = airoha_qdma_rx_process(q, budget - done);
+- done += cur;
+- } while (cur && done < budget);
+-
+- if (done < budget && napi_complete(napi))
+- airoha_qdma_irq_enable(q->qdma, QDMA_INT_REG_IDX1,
+- RX_DONE_INT_MASK);
+-
+- return done;
+-}
+-
+-static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
+- struct airoha_qdma *qdma, int ndesc)
+-{
+- const struct page_pool_params pp_params = {
+- .order = 0,
+- .pool_size = 256,
+- .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+- .dma_dir = DMA_FROM_DEVICE,
+- .max_len = PAGE_SIZE,
+- .nid = NUMA_NO_NODE,
+- .dev = qdma->eth->dev,
+- .napi = &q->napi,
+- };
+- struct airoha_eth *eth = qdma->eth;
+- int qid = q - &qdma->q_rx[0], thr;
+- dma_addr_t dma_addr;
+-
+- q->buf_size = PAGE_SIZE / 2;
+- q->qdma = qdma;
+-
+- q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
+- GFP_KERNEL);
+- if (!q->entry)
+- return -ENOMEM;
+-
+- q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc),
+- &dma_addr, GFP_KERNEL);
+- if (!q->desc)
+- return -ENOMEM;
+-
+- q->page_pool = page_pool_create(&pp_params);
+- if (IS_ERR(q->page_pool)) {
+- int err = PTR_ERR(q->page_pool);
+-
+- q->page_pool = NULL;
+- return err;
+- }
+-
+- q->ndesc = ndesc;
+- netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
+-
+- airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);
+- airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),
+- RX_RING_SIZE_MASK,
+- FIELD_PREP(RX_RING_SIZE_MASK, ndesc));
+-
+- thr = clamp(ndesc >> 3, 1, 32);
+- airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK,
+- FIELD_PREP(RX_RING_THR_MASK, thr));
+- airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
+- FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head));
+-
+- airoha_qdma_fill_rx_queue(q);
+-
+- return 0;
+-}
+-
+-static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
+-{
+- struct airoha_qdma *qdma = q->qdma;
+- struct airoha_eth *eth = qdma->eth;
+- int qid = q - &qdma->q_rx[0];
+-
+- while (q->queued) {
+- struct airoha_queue_entry *e = &q->entry[q->tail];
+- struct airoha_qdma_desc *desc = &q->desc[q->tail];
+- struct page *page = virt_to_head_page(e->buf);
+-
+- dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
+- page_pool_get_dma_dir(q->page_pool));
+- page_pool_put_full_page(q->page_pool, page, false);
+- /* Reset DMA descriptor */
+- WRITE_ONCE(desc->ctrl, 0);
+- WRITE_ONCE(desc->addr, 0);
+- WRITE_ONCE(desc->data, 0);
+- WRITE_ONCE(desc->msg0, 0);
+- WRITE_ONCE(desc->msg1, 0);
+- WRITE_ONCE(desc->msg2, 0);
+- WRITE_ONCE(desc->msg3, 0);
+-
+- q->tail = (q->tail + 1) % q->ndesc;
+- q->queued--;
+- }
+-
+- q->head = q->tail;
+- /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is
+- * empty.
+- */
+- airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK,
+- FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
+- airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
+- FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail));
+-}
+-
+-static int airoha_qdma_init_rx(struct airoha_qdma *qdma)
+-{
+- int i;
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+- int err;
+-
+- if (!(RX_DONE_INT_MASK & BIT(i))) {
+- /* rx-queue not binded to irq */
+- continue;
+- }
+-
+- err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma,
+- RX_DSCP_NUM(i));
+- if (err)
+- return err;
+- }
+-
+- return 0;
+-}
+-
+-static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
+-{
+- struct airoha_tx_irq_queue *irq_q;
+- int id, done = 0, irq_queued;
+- struct airoha_qdma *qdma;
+- struct airoha_eth *eth;
+- u32 status, head;
+-
+- irq_q = container_of(napi, struct airoha_tx_irq_queue, napi);
+- qdma = irq_q->qdma;
+- id = irq_q - &qdma->q_tx_irq[0];
+- eth = qdma->eth;
+-
+- status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id));
+- head = FIELD_GET(IRQ_HEAD_IDX_MASK, status);
+- head = head % irq_q->size;
+- irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status);
+-
+- while (irq_queued > 0 && done < budget) {
+- u32 qid, last, val = irq_q->q[head];
+- struct airoha_queue *q;
+-
+- if (val == 0xff)
+- break;
+-
+- irq_q->q[head] = 0xff; /* mark as done */
+- head = (head + 1) % irq_q->size;
+- irq_queued--;
+- done++;
+-
+- last = FIELD_GET(IRQ_DESC_IDX_MASK, val);
+- qid = FIELD_GET(IRQ_RING_IDX_MASK, val);
+-
+- if (qid >= ARRAY_SIZE(qdma->q_tx))
+- continue;
+-
+- q = &qdma->q_tx[qid];
+- if (!q->ndesc)
+- continue;
+-
+- spin_lock_bh(&q->lock);
+-
+- while (q->queued > 0) {
+- struct airoha_qdma_desc *desc = &q->desc[q->tail];
+- struct airoha_queue_entry *e = &q->entry[q->tail];
+- u32 desc_ctrl = le32_to_cpu(desc->ctrl);
+- struct sk_buff *skb = e->skb;
+- u16 index = q->tail;
+-
+- if (!(desc_ctrl & QDMA_DESC_DONE_MASK) &&
+- !(desc_ctrl & QDMA_DESC_DROP_MASK))
+- break;
+-
+- q->tail = (q->tail + 1) % q->ndesc;
+- q->queued--;
+-
+- dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
+- DMA_TO_DEVICE);
+-
+- WRITE_ONCE(desc->msg0, 0);
+- WRITE_ONCE(desc->msg1, 0);
+-
+- if (skb) {
+- u16 queue = skb_get_queue_mapping(skb);
+- struct netdev_queue *txq;
+-
+- txq = netdev_get_tx_queue(skb->dev, queue);
+- netdev_tx_completed_queue(txq, 1, skb->len);
+- if (netif_tx_queue_stopped(txq) &&
+- q->ndesc - q->queued >= q->free_thr)
+- netif_tx_wake_queue(txq);
+-
+- dev_kfree_skb_any(skb);
+- e->skb = NULL;
+- }
+-
+- if (index == last)
+- break;
+- }
+-
+- spin_unlock_bh(&q->lock);
+- }
+-
+- if (done) {
+- int i, len = done >> 7;
+-
+- for (i = 0; i < len; i++)
+- airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
+- IRQ_CLEAR_LEN_MASK, 0x80);
+- airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
+- IRQ_CLEAR_LEN_MASK, (done & 0x7f));
+- }
+-
+- if (done < budget && napi_complete(napi))
+- airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX0,
+- TX_DONE_INT_MASK(id));
+-
+- return done;
+-}
+-
+-static int airoha_qdma_init_tx_queue(struct airoha_queue *q,
+- struct airoha_qdma *qdma, int size)
+-{
+- struct airoha_eth *eth = qdma->eth;
+- int i, qid = q - &qdma->q_tx[0];
+- dma_addr_t dma_addr;
+-
+- spin_lock_init(&q->lock);
+- q->ndesc = size;
+- q->qdma = qdma;
+- q->free_thr = 1 + MAX_SKB_FRAGS;
+-
+- q->entry = devm_kzalloc(eth->dev, q->ndesc * sizeof(*q->entry),
+- GFP_KERNEL);
+- if (!q->entry)
+- return -ENOMEM;
+-
+- q->desc = dmam_alloc_coherent(eth->dev, q->ndesc * sizeof(*q->desc),
+- &dma_addr, GFP_KERNEL);
+- if (!q->desc)
+- return -ENOMEM;
+-
+- for (i = 0; i < q->ndesc; i++) {
+- u32 val;
+-
+- val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1);
+- WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val));
+- }
+-
+- airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr);
+- airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,
+- FIELD_PREP(TX_RING_CPU_IDX_MASK, q->head));
+- airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,
+- FIELD_PREP(TX_RING_DMA_IDX_MASK, q->head));
+-
+- return 0;
+-}
+-
+-static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q,
+- struct airoha_qdma *qdma, int size)
+-{
+- int id = irq_q - &qdma->q_tx_irq[0];
+- struct airoha_eth *eth = qdma->eth;
+- dma_addr_t dma_addr;
+-
+- netif_napi_add_tx(eth->napi_dev, &irq_q->napi,
+- airoha_qdma_tx_napi_poll);
+- irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32),
+- &dma_addr, GFP_KERNEL);
+- if (!irq_q->q)
+- return -ENOMEM;
+-
+- memset(irq_q->q, 0xff, size * sizeof(u32));
+- irq_q->size = size;
+- irq_q->qdma = qdma;
+-
+- airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr);
+- airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK,
+- FIELD_PREP(TX_IRQ_DEPTH_MASK, size));
+- airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK,
+- FIELD_PREP(TX_IRQ_THR_MASK, 1));
+-
+- return 0;
+-}
+-
+-static int airoha_qdma_init_tx(struct airoha_qdma *qdma)
+-{
+- int i, err;
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
+- err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma,
+- IRQ_QUEUE_LEN(i));
+- if (err)
+- return err;
+- }
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
+- err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma,
+- TX_DSCP_NUM);
+- if (err)
+- return err;
+- }
+-
+- return 0;
+-}
+-
+-static void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q)
+-{
+- struct airoha_eth *eth = q->qdma->eth;
+-
+- spin_lock_bh(&q->lock);
+- while (q->queued) {
+- struct airoha_queue_entry *e = &q->entry[q->tail];
+-
+- dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
+- DMA_TO_DEVICE);
+- dev_kfree_skb_any(e->skb);
+- e->skb = NULL;
+-
+- q->tail = (q->tail + 1) % q->ndesc;
+- q->queued--;
+- }
+- spin_unlock_bh(&q->lock);
+-}
+-
+-static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma)
+-{
+- struct airoha_eth *eth = qdma->eth;
+- dma_addr_t dma_addr;
+- u32 status;
+- int size;
+-
+- size = HW_DSCP_NUM * sizeof(struct airoha_qdma_fwd_desc);
+- qdma->hfwd.desc = dmam_alloc_coherent(eth->dev, size, &dma_addr,
+- GFP_KERNEL);
+- if (!qdma->hfwd.desc)
+- return -ENOMEM;
+-
+- airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr);
+-
+- size = AIROHA_MAX_PACKET_SIZE * HW_DSCP_NUM;
+- qdma->hfwd.q = dmam_alloc_coherent(eth->dev, size, &dma_addr,
+- GFP_KERNEL);
+- if (!qdma->hfwd.q)
+- return -ENOMEM;
+-
+- airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr);
+-
+- airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG,
+- HW_FWD_DSCP_PAYLOAD_SIZE_MASK,
+- FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK, 0));
+- airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK,
+- FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128));
+- airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG,
+- LMGR_INIT_START | LMGR_SRAM_MODE_MASK |
+- HW_FWD_DESC_NUM_MASK,
+- FIELD_PREP(HW_FWD_DESC_NUM_MASK, HW_DSCP_NUM) |
+- LMGR_INIT_START);
+-
+- return read_poll_timeout(airoha_qdma_rr, status,
+- !(status & LMGR_INIT_START), USEC_PER_MSEC,
+- 30 * USEC_PER_MSEC, true, qdma,
+- REG_LMGR_INIT_CFG);
+-}
+-
+-static void airoha_qdma_init_qos(struct airoha_qdma *qdma)
+-{
+- airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK);
+- airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK);
+-
+- airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG,
+- PSE_BUF_ESTIMATE_EN_MASK);
+-
+- airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG,
+- EGRESS_RATE_METER_EN_MASK |
+- EGRESS_RATE_METER_EQ_RATE_EN_MASK);
+- /* 2047us x 31 = 63.457ms */
+- airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
+- EGRESS_RATE_METER_WINDOW_SZ_MASK,
+- FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK, 0x1f));
+- airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
+- EGRESS_RATE_METER_TIMESLICE_MASK,
+- FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK, 0x7ff));
+-
+- /* ratelimit init */
+- airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK);
+- /* fast-tick 25us */
+- airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK,
+- FIELD_PREP(GLB_FAST_TICK_MASK, 25));
+- airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK,
+- FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40));
+-
+- airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK);
+- airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK,
+- FIELD_PREP(EGRESS_FAST_TICK_MASK, 25));
+- airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG,
+- EGRESS_SLOW_TICK_RATIO_MASK,
+- FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40));
+-
+- airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK);
+- airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG,
+- INGRESS_TRTCM_MODE_MASK);
+- airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK,
+- FIELD_PREP(INGRESS_FAST_TICK_MASK, 125));
+- airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG,
+- INGRESS_SLOW_TICK_RATIO_MASK,
+- FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8));
+-
+- airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK);
+- airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK,
+- FIELD_PREP(SLA_FAST_TICK_MASK, 25));
+- airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK,
+- FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40));
+-}
+-
+-static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma)
+-{
+- int i;
+-
+- for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) {
+- /* Tx-cpu transferred count */
+- airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0);
+- airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1),
+- CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
+- CNTR_ALL_DSCP_RING_EN_MASK |
+- FIELD_PREP(CNTR_CHAN_MASK, i));
+- /* Tx-fwd transferred count */
+- airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0);
+- airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1),
+- CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
+- CNTR_ALL_DSCP_RING_EN_MASK |
+- FIELD_PREP(CNTR_SRC_MASK, 1) |
+- FIELD_PREP(CNTR_CHAN_MASK, i));
+- }
+-}
+-
+-static int airoha_qdma_hw_init(struct airoha_qdma *qdma)
+-{
+- int i;
+-
+- /* clear pending irqs */
+- for (i = 0; i < ARRAY_SIZE(qdma->irqmask); i++)
+- airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff);
+-
+- /* setup irqs */
+- airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX0, INT_IDX0_MASK);
+- airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX1, INT_IDX1_MASK);
+- airoha_qdma_irq_enable(qdma, QDMA_INT_REG_IDX4, INT_IDX4_MASK);
+-
+- /* setup irq binding */
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
+- if (!qdma->q_tx[i].ndesc)
+- continue;
+-
+- if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i))
+- airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i),
+- TX_RING_IRQ_BLOCKING_CFG_MASK);
+- else
+- airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i),
+- TX_RING_IRQ_BLOCKING_CFG_MASK);
+- }
+-
+- airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG,
+- GLOBAL_CFG_RX_2B_OFFSET_MASK |
+- FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) |
+- GLOBAL_CFG_CPU_TXR_RR_MASK |
+- GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK |
+- GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK |
+- GLOBAL_CFG_MULTICAST_EN_MASK |
+- GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK |
+- GLOBAL_CFG_TX_WB_DONE_MASK |
+- FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2));
+-
+- airoha_qdma_init_qos(qdma);
+-
+- /* disable qdma rx delay interrupt */
+- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+- if (!qdma->q_rx[i].ndesc)
+- continue;
+-
+- airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i),
+- RX_DELAY_INT_MASK);
+- }
+-
+- airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG,
+- TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN);
+- airoha_qdma_init_qos_stats(qdma);
+-
+- return 0;
+-}
+-
+-static irqreturn_t airoha_irq_handler(int irq, void *dev_instance)
+-{
+- struct airoha_qdma *qdma = dev_instance;
+- u32 intr[ARRAY_SIZE(qdma->irqmask)];
+- int i;
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->irqmask); i++) {
+- intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i));
+- intr[i] &= qdma->irqmask[i];
+- airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]);
+- }
+-
+- if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state))
+- return IRQ_NONE;
+-
+- if (intr[1] & RX_DONE_INT_MASK) {
+- airoha_qdma_irq_disable(qdma, QDMA_INT_REG_IDX1,
+- RX_DONE_INT_MASK);
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+- if (!qdma->q_rx[i].ndesc)
+- continue;
+-
+- if (intr[1] & BIT(i))
+- napi_schedule(&qdma->q_rx[i].napi);
+- }
+- }
+-
+- if (intr[0] & INT_TX_MASK) {
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
+- if (!(intr[0] & TX_DONE_INT_MASK(i)))
+- continue;
+-
+- airoha_qdma_irq_disable(qdma, QDMA_INT_REG_IDX0,
+- TX_DONE_INT_MASK(i));
+- napi_schedule(&qdma->q_tx_irq[i].napi);
+- }
+- }
+-
+- return IRQ_HANDLED;
+-}
+-
+-static int airoha_qdma_init(struct platform_device *pdev,
+- struct airoha_eth *eth,
+- struct airoha_qdma *qdma)
+-{
+- int err, id = qdma - ð->qdma[0];
+- const char *res;
+-
+- spin_lock_init(&qdma->irq_lock);
+- qdma->eth = eth;
+-
+- res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id);
+- if (!res)
+- return -ENOMEM;
+-
+- qdma->regs = devm_platform_ioremap_resource_byname(pdev, res);
+- if (IS_ERR(qdma->regs))
+- return dev_err_probe(eth->dev, PTR_ERR(qdma->regs),
+- "failed to iomap qdma%d regs\n", id);
+-
+- qdma->irq = platform_get_irq(pdev, 4 * id);
+- if (qdma->irq < 0)
+- return qdma->irq;
+-
+- err = devm_request_irq(eth->dev, qdma->irq, airoha_irq_handler,
+- IRQF_SHARED, KBUILD_MODNAME, qdma);
+- if (err)
+- return err;
+-
+- err = airoha_qdma_init_rx(qdma);
+- if (err)
+- return err;
+-
+- err = airoha_qdma_init_tx(qdma);
+- if (err)
+- return err;
+-
+- err = airoha_qdma_init_hfwd_queues(qdma);
+- if (err)
+- return err;
+-
+- return airoha_qdma_hw_init(qdma);
+-}
+-
+-static int airoha_hw_init(struct platform_device *pdev,
+- struct airoha_eth *eth)
+-{
+- int err, i;
+-
+- /* disable xsi */
+- err = reset_control_bulk_assert(ARRAY_SIZE(eth->xsi_rsts),
+- eth->xsi_rsts);
+- if (err)
+- return err;
+-
+- err = reset_control_bulk_assert(ARRAY_SIZE(eth->rsts), eth->rsts);
+- if (err)
+- return err;
+-
+- msleep(20);
+- err = reset_control_bulk_deassert(ARRAY_SIZE(eth->rsts), eth->rsts);
+- if (err)
+- return err;
+-
+- msleep(20);
+- err = airoha_fe_init(eth);
+- if (err)
+- return err;
+-
+- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
+- err = airoha_qdma_init(pdev, eth, ð->qdma[i]);
+- if (err)
+- return err;
+- }
+-
+- set_bit(DEV_STATE_INITIALIZED, ð->state);
+-
+- return 0;
+-}
+-
+-static void airoha_hw_cleanup(struct airoha_qdma *qdma)
+-{
+- int i;
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+- if (!qdma->q_rx[i].ndesc)
+- continue;
+-
+- netif_napi_del(&qdma->q_rx[i].napi);
+- airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]);
+- if (qdma->q_rx[i].page_pool)
+- page_pool_destroy(qdma->q_rx[i].page_pool);
+- }
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
+- netif_napi_del(&qdma->q_tx_irq[i].napi);
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
+- if (!qdma->q_tx[i].ndesc)
+- continue;
+-
+- airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);
+- }
+-}
+-
+-static void airoha_qdma_start_napi(struct airoha_qdma *qdma)
+-{
+- int i;
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
+- napi_enable(&qdma->q_tx_irq[i].napi);
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+- if (!qdma->q_rx[i].ndesc)
+- continue;
+-
+- napi_enable(&qdma->q_rx[i].napi);
+- }
+-}
+-
+-static void airoha_qdma_stop_napi(struct airoha_qdma *qdma)
+-{
+- int i;
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
+- napi_disable(&qdma->q_tx_irq[i].napi);
+-
+- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+- if (!qdma->q_rx[i].ndesc)
+- continue;
+-
+- napi_disable(&qdma->q_rx[i].napi);
+- }
+-}
+-
+-static void airoha_update_hw_stats(struct airoha_gdm_port *port)
+-{
+- struct airoha_eth *eth = port->qdma->eth;
+- u32 val, i = 0;
+-
+- spin_lock(&port->stats.lock);
+- u64_stats_update_begin(&port->stats.syncp);
+-
+- /* TX */
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_H(port->id));
+- port->stats.tx_ok_pkts += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_PKT_CNT_L(port->id));
+- port->stats.tx_ok_pkts += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_H(port->id));
+- port->stats.tx_ok_bytes += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_OK_BYTE_CNT_L(port->id));
+- port->stats.tx_ok_bytes += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_DROP_CNT(port->id));
+- port->stats.tx_drops += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_BC_CNT(port->id));
+- port->stats.tx_broadcast += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_MC_CNT(port->id));
+- port->stats.tx_multicast += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_RUNT_CNT(port->id));
+- port->stats.tx_len[i] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_H(port->id));
+- port->stats.tx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_E64_CNT_L(port->id));
+- port->stats.tx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_H(port->id));
+- port->stats.tx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L64_CNT_L(port->id));
+- port->stats.tx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_H(port->id));
+- port->stats.tx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L127_CNT_L(port->id));
+- port->stats.tx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_H(port->id));
+- port->stats.tx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L255_CNT_L(port->id));
+- port->stats.tx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_H(port->id));
+- port->stats.tx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L511_CNT_L(port->id));
+- port->stats.tx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_H(port->id));
+- port->stats.tx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_L1023_CNT_L(port->id));
+- port->stats.tx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_TX_ETH_LONG_CNT(port->id));
+- port->stats.tx_len[i++] += val;
+-
+- /* RX */
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_H(port->id));
+- port->stats.rx_ok_pkts += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_PKT_CNT_L(port->id));
+- port->stats.rx_ok_pkts += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_H(port->id));
+- port->stats.rx_ok_bytes += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_OK_BYTE_CNT_L(port->id));
+- port->stats.rx_ok_bytes += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_DROP_CNT(port->id));
+- port->stats.rx_drops += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_BC_CNT(port->id));
+- port->stats.rx_broadcast += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_MC_CNT(port->id));
+- port->stats.rx_multicast += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ERROR_DROP_CNT(port->id));
+- port->stats.rx_errors += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_CRC_ERR_CNT(port->id));
+- port->stats.rx_crc_error += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_OVERFLOW_DROP_CNT(port->id));
+- port->stats.rx_over_errors += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_FRAG_CNT(port->id));
+- port->stats.rx_fragment += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_JABBER_CNT(port->id));
+- port->stats.rx_jabber += val;
+-
+- i = 0;
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_RUNT_CNT(port->id));
+- port->stats.rx_len[i] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_H(port->id));
+- port->stats.rx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_E64_CNT_L(port->id));
+- port->stats.rx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_H(port->id));
+- port->stats.rx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L64_CNT_L(port->id));
+- port->stats.rx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_H(port->id));
+- port->stats.rx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L127_CNT_L(port->id));
+- port->stats.rx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_H(port->id));
+- port->stats.rx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L255_CNT_L(port->id));
+- port->stats.rx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_H(port->id));
+- port->stats.rx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L511_CNT_L(port->id));
+- port->stats.rx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_H(port->id));
+- port->stats.rx_len[i] += ((u64)val << 32);
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_L1023_CNT_L(port->id));
+- port->stats.rx_len[i++] += val;
+-
+- val = airoha_fe_rr(eth, REG_FE_GDM_RX_ETH_LONG_CNT(port->id));
+- port->stats.rx_len[i++] += val;
+-
+- /* reset mib counters */
+- airoha_fe_set(eth, REG_FE_GDM_MIB_CLEAR(port->id),
+- FE_GDM_MIB_RX_CLEAR_MASK | FE_GDM_MIB_TX_CLEAR_MASK);
+-
+- u64_stats_update_end(&port->stats.syncp);
+- spin_unlock(&port->stats.lock);
+-}
+-
+-static int airoha_dev_open(struct net_device *dev)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- struct airoha_qdma *qdma = port->qdma;
+- int err;
+-
+- netif_tx_start_all_queues(dev);
+- err = airoha_set_gdm_ports(qdma->eth, true);
+- if (err)
+- return err;
+-
+- if (netdev_uses_dsa(dev))
+- airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id),
+- GDM_STAG_EN_MASK);
+- else
+- airoha_fe_clear(qdma->eth, REG_GDM_INGRESS_CFG(port->id),
+- GDM_STAG_EN_MASK);
+-
+- airoha_qdma_set(qdma, REG_QDMA_GLOBAL_CFG,
+- GLOBAL_CFG_TX_DMA_EN_MASK |
+- GLOBAL_CFG_RX_DMA_EN_MASK);
+-
+- return 0;
+-}
+-
+-static int airoha_dev_stop(struct net_device *dev)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- struct airoha_qdma *qdma = port->qdma;
+- int err;
+-
+- netif_tx_disable(dev);
+- err = airoha_set_gdm_ports(qdma->eth, false);
+- if (err)
+- return err;
+-
+- airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,
+- GLOBAL_CFG_TX_DMA_EN_MASK |
+- GLOBAL_CFG_RX_DMA_EN_MASK);
+-
+- return 0;
+-}
+-
+-static int airoha_dev_set_macaddr(struct net_device *dev, void *p)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- int err;
+-
+- err = eth_mac_addr(dev, p);
+- if (err)
+- return err;
+-
+- airoha_set_macaddr(port, dev->dev_addr);
+-
+- return 0;
+-}
+-
+-static int airoha_dev_init(struct net_device *dev)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+-
+- airoha_set_macaddr(port, dev->dev_addr);
+-
+- return 0;
+-}
+-
+-static void airoha_dev_get_stats64(struct net_device *dev,
+- struct rtnl_link_stats64 *storage)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- unsigned int start;
+-
+- airoha_update_hw_stats(port);
+- do {
+- start = u64_stats_fetch_begin(&port->stats.syncp);
+- storage->rx_packets = port->stats.rx_ok_pkts;
+- storage->tx_packets = port->stats.tx_ok_pkts;
+- storage->rx_bytes = port->stats.rx_ok_bytes;
+- storage->tx_bytes = port->stats.tx_ok_bytes;
+- storage->multicast = port->stats.rx_multicast;
+- storage->rx_errors = port->stats.rx_errors;
+- storage->rx_dropped = port->stats.rx_drops;
+- storage->tx_dropped = port->stats.tx_drops;
+- storage->rx_crc_errors = port->stats.rx_crc_error;
+- storage->rx_over_errors = port->stats.rx_over_errors;
+- } while (u64_stats_fetch_retry(&port->stats.syncp, start));
+-}
+-
+-static u16 airoha_dev_select_queue(struct net_device *dev, struct sk_buff *skb,
+- struct net_device *sb_dev)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- int queue, channel;
+-
+- /* For dsa device select QoS channel according to the dsa user port
+- * index, rely on port id otherwise. Select QoS queue based on the
+- * skb priority.
+- */
+- channel = netdev_uses_dsa(dev) ? skb_get_queue_mapping(skb) : port->id;
+- channel = channel % AIROHA_NUM_QOS_CHANNELS;
+- queue = (skb->priority - 1) % AIROHA_NUM_QOS_QUEUES; /* QoS queue */
+- queue = channel * AIROHA_NUM_QOS_QUEUES + queue;
+-
+- return queue < dev->num_tx_queues ? queue : 0;
+-}
+-
+-static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
+- struct net_device *dev)
+-{
+- struct skb_shared_info *sinfo = skb_shinfo(skb);
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- u32 msg0, msg1, len = skb_headlen(skb);
+- struct airoha_qdma *qdma = port->qdma;
+- u32 nr_frags = 1 + sinfo->nr_frags;
+- struct netdev_queue *txq;
+- struct airoha_queue *q;
+- void *data = skb->data;
+- int i, qid;
+- u16 index;
+- u8 fport;
+-
+- qid = skb_get_queue_mapping(skb) % ARRAY_SIZE(qdma->q_tx);
+- msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
+- qid / AIROHA_NUM_QOS_QUEUES) |
+- FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
+- qid % AIROHA_NUM_QOS_QUEUES);
+- if (skb->ip_summed == CHECKSUM_PARTIAL)
+- msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TCO_MASK, 1) |
+- FIELD_PREP(QDMA_ETH_TXMSG_UCO_MASK, 1) |
+- FIELD_PREP(QDMA_ETH_TXMSG_ICO_MASK, 1);
+-
+- /* TSO: fill MSS info in tcp checksum field */
+- if (skb_is_gso(skb)) {
+- if (skb_cow_head(skb, 0))
+- goto error;
+-
+- if (sinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) {
+- __be16 csum = cpu_to_be16(sinfo->gso_size);
+-
+- tcp_hdr(skb)->check = (__force __sum16)csum;
+- msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TSO_MASK, 1);
+- }
+- }
+-
+- fport = port->id == 4 ? FE_PSE_PORT_GDM4 : port->id;
+- msg1 = FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
+- FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
+-
+- q = &qdma->q_tx[qid];
+- if (WARN_ON_ONCE(!q->ndesc))
+- goto error;
+-
+- spin_lock_bh(&q->lock);
+-
+- txq = netdev_get_tx_queue(dev, qid);
+- if (q->queued + nr_frags > q->ndesc) {
+- /* not enough space in the queue */
+- netif_tx_stop_queue(txq);
+- spin_unlock_bh(&q->lock);
+- return NETDEV_TX_BUSY;
+- }
+-
+- index = q->head;
+- for (i = 0; i < nr_frags; i++) {
+- struct airoha_qdma_desc *desc = &q->desc[index];
+- struct airoha_queue_entry *e = &q->entry[index];
+- skb_frag_t *frag = &sinfo->frags[i];
+- dma_addr_t addr;
+- u32 val;
+-
+- addr = dma_map_single(dev->dev.parent, data, len,
+- DMA_TO_DEVICE);
+- if (unlikely(dma_mapping_error(dev->dev.parent, addr)))
+- goto error_unmap;
+-
+- index = (index + 1) % q->ndesc;
+-
+- val = FIELD_PREP(QDMA_DESC_LEN_MASK, len);
+- if (i < nr_frags - 1)
+- val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1);
+- WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
+- WRITE_ONCE(desc->addr, cpu_to_le32(addr));
+- val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, index);
+- WRITE_ONCE(desc->data, cpu_to_le32(val));
+- WRITE_ONCE(desc->msg0, cpu_to_le32(msg0));
+- WRITE_ONCE(desc->msg1, cpu_to_le32(msg1));
+- WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
+-
+- e->skb = i ? NULL : skb;
+- e->dma_addr = addr;
+- e->dma_len = len;
+-
+- data = skb_frag_address(frag);
+- len = skb_frag_size(frag);
+- }
+-
+- q->head = index;
+- q->queued += i;
+-
+- skb_tx_timestamp(skb);
+- netdev_tx_sent_queue(txq, skb->len);
+-
+- if (netif_xmit_stopped(txq) || !netdev_xmit_more())
+- airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
+- TX_RING_CPU_IDX_MASK,
+- FIELD_PREP(TX_RING_CPU_IDX_MASK, q->head));
+-
+- if (q->ndesc - q->queued < q->free_thr)
+- netif_tx_stop_queue(txq);
+-
+- spin_unlock_bh(&q->lock);
+-
+- return NETDEV_TX_OK;
+-
+-error_unmap:
+- for (i--; i >= 0; i--) {
+- index = (q->head + i) % q->ndesc;
+- dma_unmap_single(dev->dev.parent, q->entry[index].dma_addr,
+- q->entry[index].dma_len, DMA_TO_DEVICE);
+- }
+-
+- spin_unlock_bh(&q->lock);
+-error:
+- dev_kfree_skb_any(skb);
+- dev->stats.tx_dropped++;
+-
+- return NETDEV_TX_OK;
+-}
+-
+-static void airoha_ethtool_get_drvinfo(struct net_device *dev,
+- struct ethtool_drvinfo *info)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- struct airoha_eth *eth = port->qdma->eth;
+-
+- strscpy(info->driver, eth->dev->driver->name, sizeof(info->driver));
+- strscpy(info->bus_info, dev_name(eth->dev), sizeof(info->bus_info));
+-}
+-
+-static void airoha_ethtool_get_mac_stats(struct net_device *dev,
+- struct ethtool_eth_mac_stats *stats)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- unsigned int start;
+-
+- airoha_update_hw_stats(port);
+- do {
+- start = u64_stats_fetch_begin(&port->stats.syncp);
+- stats->MulticastFramesXmittedOK = port->stats.tx_multicast;
+- stats->BroadcastFramesXmittedOK = port->stats.tx_broadcast;
+- stats->BroadcastFramesReceivedOK = port->stats.rx_broadcast;
+- } while (u64_stats_fetch_retry(&port->stats.syncp, start));
+-}
+-
+-static const struct ethtool_rmon_hist_range airoha_ethtool_rmon_ranges[] = {
+- { 0, 64 },
+- { 65, 127 },
+- { 128, 255 },
+- { 256, 511 },
+- { 512, 1023 },
+- { 1024, 1518 },
+- { 1519, 10239 },
+- {},
+-};
+-
+-static void
+-airoha_ethtool_get_rmon_stats(struct net_device *dev,
+- struct ethtool_rmon_stats *stats,
+- const struct ethtool_rmon_hist_range **ranges)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+- struct airoha_hw_stats *hw_stats = &port->stats;
+- unsigned int start;
+-
+- BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) !=
+- ARRAY_SIZE(hw_stats->tx_len) + 1);
+- BUILD_BUG_ON(ARRAY_SIZE(airoha_ethtool_rmon_ranges) !=
+- ARRAY_SIZE(hw_stats->rx_len) + 1);
+-
+- *ranges = airoha_ethtool_rmon_ranges;
+- airoha_update_hw_stats(port);
+- do {
+- int i;
+-
+- start = u64_stats_fetch_begin(&port->stats.syncp);
+- stats->fragments = hw_stats->rx_fragment;
+- stats->jabbers = hw_stats->rx_jabber;
+- for (i = 0; i < ARRAY_SIZE(airoha_ethtool_rmon_ranges) - 1;
+- i++) {
+- stats->hist[i] = hw_stats->rx_len[i];
+- stats->hist_tx[i] = hw_stats->tx_len[i];
+- }
+- } while (u64_stats_fetch_retry(&port->stats.syncp, start));
+-}
+-
+-static int airoha_qdma_set_chan_tx_sched(struct airoha_gdm_port *port,
+- int channel, enum tx_sched_mode mode,
+- const u16 *weights, u8 n_weights)
+-{
+- int i;
+-
+- for (i = 0; i < AIROHA_NUM_TX_RING; i++)
+- airoha_qdma_clear(port->qdma, REG_QUEUE_CLOSE_CFG(channel),
+- TXQ_DISABLE_CHAN_QUEUE_MASK(channel, i));
+-
+- for (i = 0; i < n_weights; i++) {
+- u32 status;
+- int err;
+-
+- airoha_qdma_wr(port->qdma, REG_TXWRR_WEIGHT_CFG,
+- TWRR_RW_CMD_MASK |
+- FIELD_PREP(TWRR_CHAN_IDX_MASK, channel) |
+- FIELD_PREP(TWRR_QUEUE_IDX_MASK, i) |
+- FIELD_PREP(TWRR_VALUE_MASK, weights[i]));
+- err = read_poll_timeout(airoha_qdma_rr, status,
+- status & TWRR_RW_CMD_DONE,
+- USEC_PER_MSEC, 10 * USEC_PER_MSEC,
+- true, port->qdma,
+- REG_TXWRR_WEIGHT_CFG);
+- if (err)
+- return err;
+- }
+-
+- airoha_qdma_rmw(port->qdma, REG_CHAN_QOS_MODE(channel >> 3),
+- CHAN_QOS_MODE_MASK(channel),
+- mode << __ffs(CHAN_QOS_MODE_MASK(channel)));
+-
+- return 0;
+-}
+-
+-static int airoha_qdma_set_tx_prio_sched(struct airoha_gdm_port *port,
+- int channel)
+-{
+- static const u16 w[AIROHA_NUM_QOS_QUEUES] = {};
+-
+- return airoha_qdma_set_chan_tx_sched(port, channel, TC_SCH_SP, w,
+- ARRAY_SIZE(w));
+-}
+-
+-static int airoha_qdma_set_tx_ets_sched(struct airoha_gdm_port *port,
+- int channel,
+- struct tc_ets_qopt_offload *opt)
+-{
+- struct tc_ets_qopt_offload_replace_params *p = &opt->replace_params;
+- enum tx_sched_mode mode = TC_SCH_SP;
+- u16 w[AIROHA_NUM_QOS_QUEUES] = {};
+- int i, nstrict = 0;
+-
+- if (p->bands > AIROHA_NUM_QOS_QUEUES)
+- return -EINVAL;
+-
+- for (i = 0; i < p->bands; i++) {
+- if (!p->quanta[i])
+- nstrict++;
+- }
+-
+- /* this configuration is not supported by the hw */
+- if (nstrict == AIROHA_NUM_QOS_QUEUES - 1)
+- return -EINVAL;
+-
+- for (i = 0; i < p->bands - nstrict; i++)
+- w[i] = p->weights[nstrict + i];
+-
+- if (!nstrict)
+- mode = TC_SCH_WRR8;
+- else if (nstrict < AIROHA_NUM_QOS_QUEUES - 1)
+- mode = nstrict + 1;
+-
+- return airoha_qdma_set_chan_tx_sched(port, channel, mode, w,
+- ARRAY_SIZE(w));
+-}
+-
+-static int airoha_qdma_get_tx_ets_stats(struct airoha_gdm_port *port,
+- int channel,
+- struct tc_ets_qopt_offload *opt)
+-{
+- u64 cpu_tx_packets = airoha_qdma_rr(port->qdma,
+- REG_CNTR_VAL(channel << 1));
+- u64 fwd_tx_packets = airoha_qdma_rr(port->qdma,
+- REG_CNTR_VAL((channel << 1) + 1));
+- u64 tx_packets = (cpu_tx_packets - port->cpu_tx_packets) +
+- (fwd_tx_packets - port->fwd_tx_packets);
+- _bstats_update(opt->stats.bstats, 0, tx_packets);
+-
+- port->cpu_tx_packets = cpu_tx_packets;
+- port->fwd_tx_packets = fwd_tx_packets;
+-
+- return 0;
+-}
+-
+-static int airoha_tc_setup_qdisc_ets(struct airoha_gdm_port *port,
+- struct tc_ets_qopt_offload *opt)
+-{
+- int channel;
+-
+- if (opt->parent == TC_H_ROOT)
+- return -EINVAL;
+-
+- channel = TC_H_MAJ(opt->handle) >> 16;
+- channel = channel % AIROHA_NUM_QOS_CHANNELS;
+-
+- switch (opt->command) {
+- case TC_ETS_REPLACE:
+- return airoha_qdma_set_tx_ets_sched(port, channel, opt);
+- case TC_ETS_DESTROY:
+- /* PRIO is default qdisc scheduler */
+- return airoha_qdma_set_tx_prio_sched(port, channel);
+- case TC_ETS_STATS:
+- return airoha_qdma_get_tx_ets_stats(port, channel, opt);
+- default:
+- return -EOPNOTSUPP;
+- }
+-}
+-
+-static int airoha_dev_tc_setup(struct net_device *dev, enum tc_setup_type type,
+- void *type_data)
+-{
+- struct airoha_gdm_port *port = netdev_priv(dev);
+-
+- switch (type) {
+- case TC_SETUP_QDISC_ETS:
+- return airoha_tc_setup_qdisc_ets(port, type_data);
+- default:
+- return -EOPNOTSUPP;
+- }
+-}
+-
+-static const struct net_device_ops airoha_netdev_ops = {
+- .ndo_init = airoha_dev_init,
+- .ndo_open = airoha_dev_open,
+- .ndo_stop = airoha_dev_stop,
+- .ndo_select_queue = airoha_dev_select_queue,
+- .ndo_start_xmit = airoha_dev_xmit,
+- .ndo_get_stats64 = airoha_dev_get_stats64,
+- .ndo_set_mac_address = airoha_dev_set_macaddr,
+- .ndo_setup_tc = airoha_dev_tc_setup,
+-};
+-
+-static const struct ethtool_ops airoha_ethtool_ops = {
+- .get_drvinfo = airoha_ethtool_get_drvinfo,
+- .get_eth_mac_stats = airoha_ethtool_get_mac_stats,
+- .get_rmon_stats = airoha_ethtool_get_rmon_stats,
+-};
+-
+-static int airoha_alloc_gdm_port(struct airoha_eth *eth, struct device_node *np)
+-{
+- const __be32 *id_ptr = of_get_property(np, "reg", NULL);
+- struct airoha_gdm_port *port;
+- struct airoha_qdma *qdma;
+- struct net_device *dev;
+- int err, index;
+- u32 id;
+-
+- if (!id_ptr) {
+- dev_err(eth->dev, "missing gdm port id\n");
+- return -EINVAL;
+- }
+-
+- id = be32_to_cpup(id_ptr);
+- index = id - 1;
+-
+- if (!id || id > ARRAY_SIZE(eth->ports)) {
+- dev_err(eth->dev, "invalid gdm port id: %d\n", id);
+- return -EINVAL;
+- }
+-
+- if (eth->ports[index]) {
+- dev_err(eth->dev, "duplicate gdm port id: %d\n", id);
+- return -EINVAL;
+- }
+-
+- dev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*port),
+- AIROHA_NUM_TX_RING, AIROHA_NUM_RX_RING);
+- if (!dev) {
+- dev_err(eth->dev, "alloc_etherdev failed\n");
+- return -ENOMEM;
+- }
+-
+- qdma = ð->qdma[index % AIROHA_MAX_NUM_QDMA];
+- dev->netdev_ops = &airoha_netdev_ops;
+- dev->ethtool_ops = &airoha_ethtool_ops;
+- dev->max_mtu = AIROHA_MAX_MTU;
+- dev->watchdog_timeo = 5 * HZ;
+- dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
+- NETIF_F_TSO6 | NETIF_F_IPV6_CSUM |
+- NETIF_F_SG | NETIF_F_TSO |
+- NETIF_F_HW_TC;
+- dev->features |= dev->hw_features;
+- dev->dev.of_node = np;
+- dev->irq = qdma->irq;
+- SET_NETDEV_DEV(dev, eth->dev);
+-
+- err = of_get_ethdev_address(np, dev);
+- if (err) {
+- if (err == -EPROBE_DEFER)
+- return err;
+-
+- eth_hw_addr_random(dev);
+- dev_info(eth->dev, "generated random MAC address %pM\n",
+- dev->dev_addr);
+- }
+-
+- port = netdev_priv(dev);
+- u64_stats_init(&port->stats.syncp);
+- spin_lock_init(&port->stats.lock);
+- port->qdma = qdma;
+- port->dev = dev;
+- port->id = id;
+- eth->ports[index] = port;
+-
+- return register_netdev(dev);
+-}
+-
+-static int airoha_probe(struct platform_device *pdev)
+-{
+- struct device_node *np;
+- struct airoha_eth *eth;
+- int i, err;
+-
+- eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL);
+- if (!eth)
+- return -ENOMEM;
+-
+- eth->dev = &pdev->dev;
+-
+- err = dma_set_mask_and_coherent(eth->dev, DMA_BIT_MASK(32));
+- if (err) {
+- dev_err(eth->dev, "failed configuring DMA mask\n");
+- return err;
+- }
+-
+- eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe");
+- if (IS_ERR(eth->fe_regs))
+- return dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs),
+- "failed to iomap fe regs\n");
+-
+- eth->rsts[0].id = "fe";
+- eth->rsts[1].id = "pdma";
+- eth->rsts[2].id = "qdma";
+- err = devm_reset_control_bulk_get_exclusive(eth->dev,
+- ARRAY_SIZE(eth->rsts),
+- eth->rsts);
+- if (err) {
+- dev_err(eth->dev, "failed to get bulk reset lines\n");
+- return err;
+- }
+-
+- eth->xsi_rsts[0].id = "xsi-mac";
+- eth->xsi_rsts[1].id = "hsi0-mac";
+- eth->xsi_rsts[2].id = "hsi1-mac";
+- eth->xsi_rsts[3].id = "hsi-mac";
+- eth->xsi_rsts[4].id = "xfp-mac";
+- err = devm_reset_control_bulk_get_exclusive(eth->dev,
+- ARRAY_SIZE(eth->xsi_rsts),
+- eth->xsi_rsts);
+- if (err) {
+- dev_err(eth->dev, "failed to get bulk xsi reset lines\n");
+- return err;
+- }
+-
+- eth->napi_dev = alloc_netdev_dummy(0);
+- if (!eth->napi_dev)
+- return -ENOMEM;
+-
+- /* Enable threaded NAPI by default */
+- eth->napi_dev->threaded = true;
+- strscpy(eth->napi_dev->name, "qdma_eth", sizeof(eth->napi_dev->name));
+- platform_set_drvdata(pdev, eth);
+-
+- err = airoha_hw_init(pdev, eth);
+- if (err)
+- goto error_hw_cleanup;
+-
+- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
+- airoha_qdma_start_napi(ð->qdma[i]);
+-
+- for_each_child_of_node(pdev->dev.of_node, np) {
+- if (!of_device_is_compatible(np, "airoha,eth-mac"))
+- continue;
+-
+- if (!of_device_is_available(np))
+- continue;
+-
+- err = airoha_alloc_gdm_port(eth, np);
+- if (err) {
+- of_node_put(np);
+- goto error_napi_stop;
+- }
+- }
+-
+- return 0;
+-
+-error_napi_stop:
+- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
+- airoha_qdma_stop_napi(ð->qdma[i]);
+-error_hw_cleanup:
+- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
+- airoha_hw_cleanup(ð->qdma[i]);
+-
+- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+- struct airoha_gdm_port *port = eth->ports[i];
+-
+- if (port && port->dev->reg_state == NETREG_REGISTERED)
+- unregister_netdev(port->dev);
+- }
+- free_netdev(eth->napi_dev);
+- platform_set_drvdata(pdev, NULL);
+-
+- return err;
+-}
+-
+-static void airoha_remove(struct platform_device *pdev)
+-{
+- struct airoha_eth *eth = platform_get_drvdata(pdev);
+- int i;
+-
+- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
+- airoha_qdma_stop_napi(ð->qdma[i]);
+- airoha_hw_cleanup(ð->qdma[i]);
+- }
+-
+- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+- struct airoha_gdm_port *port = eth->ports[i];
+-
+- if (!port)
+- continue;
+-
+- unregister_netdev(port->dev);
+- }
+- free_netdev(eth->napi_dev);
+-
+- platform_set_drvdata(pdev, NULL);
+-}
+-
+-static const struct of_device_id of_airoha_match[] = {
+- { .compatible = "airoha,en7581-eth" },
+- { /* sentinel */ }
+-};
+-MODULE_DEVICE_TABLE(of, of_airoha_match);
+-
+-static struct platform_driver airoha_driver = {
+- .probe = airoha_probe,
+- .remove_new = airoha_remove,
+- .driver = {
+- .name = KBUILD_MODNAME,
+- .of_match_table = of_airoha_match,
+- },
+-};
+-module_platform_driver(airoha_driver);
+-
+-MODULE_LICENSE("GPL");
+-MODULE_AUTHOR("Lorenzo Bianconi <[email protected]>");
+-MODULE_DESCRIPTION("Ethernet driver for Airoha SoC");
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c
+index 904e08de852eac..1e5522a194839c 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c
+@@ -166,9 +166,90 @@ static int mlx5_dpll_device_mode_get(const struct dpll_device *dpll,
+ return 0;
+ }
+
++enum {
++ MLX5_DPLL_SSM_CODE_PRC = 0b0010,
++ MLX5_DPLL_SSM_CODE_SSU_A = 0b0100,
++ MLX5_DPLL_SSM_CODE_SSU_B = 0b1000,
++ MLX5_DPLL_SSM_CODE_EEC1 = 0b1011,
++ MLX5_DPLL_SSM_CODE_PRTC = 0b0010,
++ MLX5_DPLL_SSM_CODE_EPRTC = 0b0010,
++ MLX5_DPLL_SSM_CODE_EEEC = 0b1011,
++ MLX5_DPLL_SSM_CODE_EPRC = 0b0010,
++};
++
++enum {
++ MLX5_DPLL_ENHANCED_SSM_CODE_PRC = 0xff,
++ MLX5_DPLL_ENHANCED_SSM_CODE_SSU_A = 0xff,
++ MLX5_DPLL_ENHANCED_SSM_CODE_SSU_B = 0xff,
++ MLX5_DPLL_ENHANCED_SSM_CODE_EEC1 = 0xff,
++ MLX5_DPLL_ENHANCED_SSM_CODE_PRTC = 0x20,
++ MLX5_DPLL_ENHANCED_SSM_CODE_EPRTC = 0x21,
++ MLX5_DPLL_ENHANCED_SSM_CODE_EEEC = 0x22,
++ MLX5_DPLL_ENHANCED_SSM_CODE_EPRC = 0x23,
++};
++
++#define __MLX5_DPLL_SSM_COMBINED_CODE(ssm_code, enhanced_ssm_code) \
++ ((ssm_code) | ((enhanced_ssm_code) << 8))
++
++#define MLX5_DPLL_SSM_COMBINED_CODE(type) \
++ __MLX5_DPLL_SSM_COMBINED_CODE(MLX5_DPLL_SSM_CODE_##type, \
++ MLX5_DPLL_ENHANCED_SSM_CODE_##type)
++
++static int mlx5_dpll_clock_quality_level_get(const struct dpll_device *dpll,
++ void *priv, unsigned long *qls,
++ struct netlink_ext_ack *extack)
++{
++ u8 network_option, ssm_code, enhanced_ssm_code;
++ u32 out[MLX5_ST_SZ_DW(msecq_reg)] = {};
++ u32 in[MLX5_ST_SZ_DW(msecq_reg)] = {};
++ struct mlx5_dpll *mdpll = priv;
++ int err;
++
++ err = mlx5_core_access_reg(mdpll->mdev, in, sizeof(in),
++ out, sizeof(out), MLX5_REG_MSECQ, 0, 0);
++ if (err)
++ return err;
++ network_option = MLX5_GET(msecq_reg, out, network_option);
++ if (network_option != 1)
++ goto errout;
++ ssm_code = MLX5_GET(msecq_reg, out, local_ssm_code);
++ enhanced_ssm_code = MLX5_GET(msecq_reg, out, local_enhanced_ssm_code);
++
++ switch (__MLX5_DPLL_SSM_COMBINED_CODE(ssm_code, enhanced_ssm_code)) {
++ case MLX5_DPLL_SSM_COMBINED_CODE(PRC):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_PRC, qls);
++ return 0;
++ case MLX5_DPLL_SSM_COMBINED_CODE(SSU_A):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_SSU_A, qls);
++ return 0;
++ case MLX5_DPLL_SSM_COMBINED_CODE(SSU_B):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_SSU_B, qls);
++ return 0;
++ case MLX5_DPLL_SSM_COMBINED_CODE(EEC1):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_EEC1, qls);
++ return 0;
++ case MLX5_DPLL_SSM_COMBINED_CODE(PRTC):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_PRTC, qls);
++ return 0;
++ case MLX5_DPLL_SSM_COMBINED_CODE(EPRTC):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_EPRTC, qls);
++ return 0;
++ case MLX5_DPLL_SSM_COMBINED_CODE(EEEC):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_EEEC, qls);
++ return 0;
++ case MLX5_DPLL_SSM_COMBINED_CODE(EPRC):
++ __set_bit(DPLL_CLOCK_QUALITY_LEVEL_ITU_OPT1_EPRC, qls);
++ return 0;
++ }
++errout:
++ NL_SET_ERR_MSG_MOD(extack, "Invalid clock quality level obtained from firmware");
++ return -EINVAL;
++}
++
+ static const struct dpll_device_ops mlx5_dpll_device_ops = {
+ .lock_status_get = mlx5_dpll_device_lock_status_get,
+ .mode_get = mlx5_dpll_device_mode_get,
++ .clock_quality_level_get = mlx5_dpll_clock_quality_level_get,
+ };
+
+ static int mlx5_dpll_pin_direction_get(const struct dpll_pin *pin,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
+index b9cf79e2712440..7a18a469961db8 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
+@@ -140,7 +140,7 @@ static int mlx5e_tc_tun_parse_vxlan_gbp_option(struct mlx5e_priv *priv,
+ gbp_mask = (u32 *)&enc_opts.mask->data[0];
+
+ if (*gbp_mask & ~VXLAN_GBP_MASK) {
+- NL_SET_ERR_MSG_FMT_MOD(extack, "Wrong VxLAN GBP mask(0x%08X)\n", *gbp_mask);
++ NL_SET_ERR_MSG_FMT_MOD(extack, "Wrong VxLAN GBP mask(0x%08X)", *gbp_mask);
+ return -EINVAL;
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+index 84e700777941ec..98ca8f486d52cd 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+@@ -158,6 +158,13 @@ static int mlx5e_dcbnl_ieee_getets(struct net_device *netdev,
+ }
+ memcpy(ets->tc_tsa, priv->dcbx.tc_tsa, sizeof(ets->tc_tsa));
+
++ /* Report 0 for non ETS TSA */
++ for (i = 0; i < ets->ets_cap; i++) {
++ if (ets->tc_tx_bw[i] == MLX5E_MAX_BW_ALLOC &&
++ priv->dcbx.tc_tsa[i] != IEEE_8021QAZ_TSA_ETS)
++ ets->tc_tx_bw[i] = 0;
++ }
++
+ return err;
+ }
+
+@@ -302,6 +309,14 @@ static int mlx5e_dbcnl_validate_ets(struct net_device *netdev,
+ }
+ }
+
++ for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
++ if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_CB_SHAPER) {
++ netdev_err(netdev,
++ "Failed to validate ETS: CB Shaper is not supported\n");
++ return -EOPNOTSUPP;
++ }
++ }
++
+ /* Validate Bandwidth Sum */
+ for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
+ if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_ETS) {
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+index 1f55c5a7edf31b..026f2867b84900 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+@@ -1912,7 +1912,7 @@ static int mlx5e_get_module_info(struct net_device *netdev,
+ int size_read = 0;
+ u8 data[4] = {0};
+
+- size_read = mlx5_query_module_eeprom(dev, 0, 2, data);
++ size_read = mlx5_query_module_eeprom(dev, 0, 2, data, NULL);
+ if (size_read < 2)
+ return -EIO;
+
+@@ -1954,6 +1954,7 @@ static int mlx5e_get_module_eeprom(struct net_device *netdev,
+ struct mlx5_core_dev *mdev = priv->mdev;
+ int offset = ee->offset;
+ int size_read;
++ u8 status = 0;
+ int i = 0;
+
+ if (!ee->len)
+@@ -1963,15 +1964,15 @@ static int mlx5e_get_module_eeprom(struct net_device *netdev,
+
+ while (i < ee->len) {
+ size_read = mlx5_query_module_eeprom(mdev, offset, ee->len - i,
+- data + i);
+-
++ data + i, &status);
+ if (!size_read)
+ /* Done reading */
+ return 0;
+
+ if (size_read < 0) {
+- netdev_err(priv->netdev, "%s: mlx5_query_eeprom failed:0x%x\n",
+- __func__, size_read);
++ netdev_err(netdev,
++ "%s: mlx5_query_eeprom failed:0x%x, status %u\n",
++ __func__, size_read, status);
+ return size_read;
+ }
+
+@@ -1991,6 +1992,7 @@ static int mlx5e_get_module_eeprom_by_page(struct net_device *netdev,
+ struct mlx5_core_dev *mdev = priv->mdev;
+ u8 *data = page_data->data;
+ int size_read;
++ u8 status = 0;
+ int i = 0;
+
+ if (!page_data->length)
+@@ -2004,7 +2006,8 @@ static int mlx5e_get_module_eeprom_by_page(struct net_device *netdev,
+ query.page = page_data->page;
+ while (i < page_data->length) {
+ query.size = page_data->length - i;
+- size_read = mlx5_query_module_eeprom_by_page(mdev, &query, data + i);
++ size_read = mlx5_query_module_eeprom_by_page(mdev, &query,
++ data + i, &status);
+
+ /* Done reading, return how many bytes was read */
+ if (!size_read)
+@@ -2013,8 +2016,8 @@ static int mlx5e_get_module_eeprom_by_page(struct net_device *netdev,
+ if (size_read < 0) {
+ NL_SET_ERR_MSG_FMT_MOD(
+ extack,
+- "Query module eeprom by page failed, read %u bytes, err %d\n",
+- i, size_read);
++ "Query module eeprom by page failed, read %u bytes, err %d, status %u",
++ i, size_read, status);
+ return size_read;
+ }
+
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+index 17f149cdcb0f25..c7eb58fd6e6af4 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+@@ -206,11 +206,11 @@ static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
+
+ static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data)
+ {
+- struct mlx5e_priv *slave_priv = my_data;
++ struct mlx5e_priv *master_priv = event_data;
+
+ switch (event) {
+ case MPV_DEVCOM_MASTER_UP:
+- mlx5_devcom_comp_set_ready(slave_priv->devcom, true);
++ mlx5_devcom_comp_set_ready(master_priv->devcom, true);
+ break;
+ case MPV_DEVCOM_MASTER_DOWN:
+ /* no need for comp set ready false since we unregister after
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c
+index d599e50af346be..fc545bfb54de76 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c
+@@ -71,7 +71,7 @@ int esw_egress_acl_vlan_create(struct mlx5_eswitch *esw,
+ flow_act.action = flow_action;
+ vport->egress.allowed_vlan =
+ mlx5_add_flow_rules(vport->egress.acl, spec,
+- &flow_act, fwd_dest, 0);
++ &flow_act, fwd_dest, fwd_dest ? 1 : 0);
+ if (IS_ERR(vport->egress.allowed_vlan)) {
+ err = PTR_ERR(vport->egress.allowed_vlan);
+ esw_warn(esw->dev,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
+index c5ea1d1d2b035c..282cef59e6173f 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
+@@ -1861,7 +1861,7 @@ int mlx5_esw_bridge_port_mdb_add(struct net_device *dev, u16 vport_num, u16 esw_
+ "Failed to lookup bridge port to add MDB (MAC=%pM,vport=%u)\n",
+ addr, vport_num);
+ NL_SET_ERR_MSG_FMT_MOD(extack,
+- "Failed to lookup bridge port to add MDB (MAC=%pM,vport=%u)\n",
++ "Failed to lookup bridge port to add MDB (MAC=%pM,vport=%u)",
+ addr, vport_num);
+ return -EINVAL;
+ }
+@@ -1874,7 +1874,7 @@ int mlx5_esw_bridge_port_mdb_add(struct net_device *dev, u16 vport_num, u16 esw_
+ "Failed to lookup bridge port vlan metadata to create MDB (MAC=%pM,vid=%u,vport=%u)\n",
+ addr, vid, vport_num);
+ NL_SET_ERR_MSG_FMT_MOD(extack,
+- "Failed to lookup vlan metadata for MDB (MAC=%pM,vid=%u,vport=%u)\n",
++ "Failed to lookup vlan metadata for MDB (MAC=%pM,vid=%u,vport=%u)",
+ addr, vid, vport_num);
+ return -EINVAL;
+ }
+@@ -1882,7 +1882,7 @@ int mlx5_esw_bridge_port_mdb_add(struct net_device *dev, u16 vport_num, u16 esw_
+
+ err = mlx5_esw_bridge_port_mdb_attach(dev, port, addr, vid);
+ if (err) {
+- NL_SET_ERR_MSG_FMT_MOD(extack, "Failed to add MDB (MAC=%pM,vid=%u,vport=%u)\n",
++ NL_SET_ERR_MSG_FMT_MOD(extack, "Failed to add MDB (MAC=%pM,vid=%u,vport=%u)",
+ addr, vid, vport_num);
+ return err;
+ }
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+index 1ca75f1815ef64..2e152f8571c7df 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
++++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+@@ -342,11 +342,11 @@ int mlx5_set_port_fcs(struct mlx5_core_dev *mdev, u8 enable);
+ void mlx5_query_port_fcs(struct mlx5_core_dev *mdev, bool *supported,
+ bool *enabled);
+ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev,
+- u16 offset, u16 size, u8 *data);
++ u16 offset, u16 size, u8 *data, u8 *status);
+ int
+ mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev,
+ struct mlx5_module_eeprom_query_params *params,
+- u8 *data);
++ u8 *data, u8 *status);
+
+ int mlx5_query_port_dcbx_param(struct mlx5_core_dev *mdev, u32 *out);
+ int mlx5_set_port_dcbx_param(struct mlx5_core_dev *mdev, u32 *in);
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c
+index 79c477e05e46cf..9981ebc64cd5bc 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/port.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c
+@@ -289,11 +289,11 @@ int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num)
+ }
+
+ static int mlx5_query_module_id(struct mlx5_core_dev *dev, int module_num,
+- u8 *module_id)
++ u8 *module_id, u8 *status)
+ {
+ u32 in[MLX5_ST_SZ_DW(mcia_reg)] = {};
+ u32 out[MLX5_ST_SZ_DW(mcia_reg)];
+- int err, status;
++ int err;
+ u8 *ptr;
+
+ MLX5_SET(mcia_reg, in, i2c_device_address, MLX5_I2C_ADDR_LOW);
+@@ -308,13 +308,13 @@ static int mlx5_query_module_id(struct mlx5_core_dev *dev, int module_num,
+ if (err)
+ return err;
+
+- status = MLX5_GET(mcia_reg, out, status);
+- if (status) {
+- mlx5_core_err(dev, "query_mcia_reg failed: status: 0x%x\n",
+- status);
++ if (MLX5_GET(mcia_reg, out, status)) {
++ if (status)
++ *status = MLX5_GET(mcia_reg, out, status);
+ return -EIO;
+ }
+- ptr = MLX5_ADDR_OF(mcia_reg, out, dword_0);
++
++ ptr = MLX5_ADDR_OF(mcia_reg, out, dwords);
+
+ *module_id = ptr[0];
+
+@@ -370,13 +370,14 @@ static int mlx5_mcia_max_bytes(struct mlx5_core_dev *dev)
+ }
+
+ static int mlx5_query_mcia(struct mlx5_core_dev *dev,
+- struct mlx5_module_eeprom_query_params *params, u8 *data)
++ struct mlx5_module_eeprom_query_params *params,
++ u8 *data, u8 *status)
+ {
+ u32 in[MLX5_ST_SZ_DW(mcia_reg)] = {};
+ u32 out[MLX5_ST_SZ_DW(mcia_reg)];
+- int status, err;
+ void *ptr;
+ u16 size;
++ int err;
+
+ size = min_t(int, params->size, mlx5_mcia_max_bytes(dev));
+
+@@ -392,21 +393,20 @@ static int mlx5_query_mcia(struct mlx5_core_dev *dev,
+ if (err)
+ return err;
+
+- status = MLX5_GET(mcia_reg, out, status);
+- if (status) {
+- mlx5_core_err(dev, "query_mcia_reg failed: status: 0x%x\n",
+- status);
++ if (MLX5_GET(mcia_reg, out, status)) {
++ if (status)
++ *status = MLX5_GET(mcia_reg, out, status);
+ return -EIO;
+ }
+
+- ptr = MLX5_ADDR_OF(mcia_reg, out, dword_0);
++ ptr = MLX5_ADDR_OF(mcia_reg, out, dwords);
+ memcpy(data, ptr, size);
+
+ return size;
+ }
+
+ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev,
+- u16 offset, u16 size, u8 *data)
++ u16 offset, u16 size, u8 *data, u8 *status)
+ {
+ struct mlx5_module_eeprom_query_params query = {0};
+ u8 module_id;
+@@ -416,7 +416,8 @@ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev,
+ if (err)
+ return err;
+
+- err = mlx5_query_module_id(dev, query.module_number, &module_id);
++ err = mlx5_query_module_id(dev, query.module_number, &module_id,
++ status);
+ if (err)
+ return err;
+
+@@ -442,12 +443,12 @@ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev,
+ query.size = size;
+ query.offset = offset;
+
+- return mlx5_query_mcia(dev, &query, data);
++ return mlx5_query_mcia(dev, &query, data, status);
+ }
+
+ int mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev,
+ struct mlx5_module_eeprom_query_params *params,
+- u8 *data)
++ u8 *data, u8 *status)
+ {
+ int err;
+
+@@ -461,7 +462,7 @@ int mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev,
+ return -EINVAL;
+ }
+
+- return mlx5_query_mcia(dev, params, data);
++ return mlx5_query_mcia(dev, params, data, status);
+ }
+
+ static int mlx5_query_port_pvlc(struct mlx5_core_dev *dev, u32 *pvlc,
+diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/mlx5hws_bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/mlx5hws_bwc.c
+index 8f3a6f9d703da4..4fdffc20cd7829 100644
+--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/mlx5hws_bwc.c
++++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/mlx5hws_bwc.c
+@@ -63,6 +63,184 @@ static void hws_bwc_matcher_init_attr(struct mlx5hws_matcher_attr *attr,
+ attr->max_num_of_at_attach = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM;
+ }
+
++static int hws_bwc_queue_poll(struct mlx5hws_context *ctx,
++ u16 queue_id,
++ u32 *pending_rules,
++ bool drain)
++{
++ struct mlx5hws_flow_op_result comp[MLX5HWS_BWC_MATCHER_REHASH_BURST_TH];
++ u16 burst_th = hws_bwc_get_burst_th(ctx, queue_id);
++ bool got_comp = *pending_rules >= burst_th;
++ bool queue_full;
++ int err = 0;
++ int ret;
++ int i;
++
++ /* Check if there are any completions at all */
++ if (!got_comp && !drain)
++ return 0;
++
++ queue_full = mlx5hws_send_engine_full(&ctx->send_queue[queue_id]);
++ while (queue_full || ((got_comp || drain) && *pending_rules)) {
++ ret = mlx5hws_send_queue_poll(ctx, queue_id, comp, burst_th);
++ if (unlikely(ret < 0)) {
++ mlx5hws_err(ctx, "BWC poll error: polling queue %d returned %d\n",
++ queue_id, ret);
++ return -EINVAL;
++ }
++
++ if (ret) {
++ (*pending_rules) -= ret;
++ for (i = 0; i < ret; i++) {
++ if (unlikely(comp[i].status != MLX5HWS_FLOW_OP_SUCCESS)) {
++ mlx5hws_err(ctx,
++ "BWC poll error: polling queue %d returned completion with error\n",
++ queue_id);
++ err = -EINVAL;
++ }
++ }
++ queue_full = false;
++ }
++
++ got_comp = !!ret;
++ }
++
++ return err;
++}
++
++static int hws_bwc_matcher_move_all_simple(struct mlx5hws_bwc_matcher *bwc_matcher)
++{
++ struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
++ u16 bwc_queues = mlx5hws_bwc_queues(ctx);
++ struct mlx5hws_bwc_rule **bwc_rules;
++ struct mlx5hws_rule_attr rule_attr;
++ u32 *pending_rules;
++ int i, j, ret = 0;
++ bool all_done;
++ u16 burst_th;
++
++ mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr);
++
++ pending_rules = kcalloc(bwc_queues, sizeof(*pending_rules), GFP_KERNEL);
++ if (!pending_rules)
++ return -ENOMEM;
++
++ bwc_rules = kcalloc(bwc_queues, sizeof(*bwc_rules), GFP_KERNEL);
++ if (!bwc_rules) {
++ ret = -ENOMEM;
++ goto free_pending_rules;
++ }
++
++ for (i = 0; i < bwc_queues; i++) {
++ if (list_empty(&bwc_matcher->rules[i]))
++ bwc_rules[i] = NULL;
++ else
++ bwc_rules[i] = list_first_entry(&bwc_matcher->rules[i],
++ struct mlx5hws_bwc_rule,
++ list_node);
++ }
++
++ do {
++ all_done = true;
++
++ for (i = 0; i < bwc_queues; i++) {
++ rule_attr.queue_id = mlx5hws_bwc_get_queue_id(ctx, i);
++ burst_th = hws_bwc_get_burst_th(ctx, rule_attr.queue_id);
++
++ for (j = 0; j < burst_th && bwc_rules[i]; j++) {
++ rule_attr.burst = !!((j + 1) % burst_th);
++ ret = mlx5hws_matcher_resize_rule_move(bwc_matcher->matcher,
++ bwc_rules[i]->rule,
++ &rule_attr);
++ if (unlikely(ret)) {
++ mlx5hws_err(ctx,
++ "Moving BWC rule failed during rehash (%d)\n",
++ ret);
++ goto free_bwc_rules;
++ }
++
++ all_done = false;
++ pending_rules[i]++;
++ bwc_rules[i] = list_is_last(&bwc_rules[i]->list_node,
++ &bwc_matcher->rules[i]) ?
++ NULL : list_next_entry(bwc_rules[i], list_node);
++
++ ret = hws_bwc_queue_poll(ctx, rule_attr.queue_id,
++ &pending_rules[i], false);
++ if (unlikely(ret))
++ goto free_bwc_rules;
++ }
++ }
++ } while (!all_done);
++
++ /* drain all the bwc queues */
++ for (i = 0; i < bwc_queues; i++) {
++ if (pending_rules[i]) {
++ u16 queue_id = mlx5hws_bwc_get_queue_id(ctx, i);
++
++ mlx5hws_send_engine_flush_queue(&ctx->send_queue[queue_id]);
++ ret = hws_bwc_queue_poll(ctx, queue_id,
++ &pending_rules[i], true);
++ if (unlikely(ret))
++ goto free_bwc_rules;
++ }
++ }
++
++free_bwc_rules:
++ kfree(bwc_rules);
++free_pending_rules:
++ kfree(pending_rules);
++
++ return ret;
++}
++
++static int hws_bwc_matcher_move_all(struct mlx5hws_bwc_matcher *bwc_matcher)
++{
++ return hws_bwc_matcher_move_all_simple(bwc_matcher);
++}
++
++static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher)
++{
++ struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
++ struct mlx5hws_matcher_attr matcher_attr = {0};
++ struct mlx5hws_matcher *old_matcher;
++ struct mlx5hws_matcher *new_matcher;
++ int ret;
++
++ hws_bwc_matcher_init_attr(&matcher_attr,
++ bwc_matcher->priority,
++ bwc_matcher->size_log);
++
++ old_matcher = bwc_matcher->matcher;
++ new_matcher = mlx5hws_matcher_create(old_matcher->tbl,
++ &bwc_matcher->mt, 1,
++ bwc_matcher->at,
++ bwc_matcher->num_of_at,
++ &matcher_attr);
++ if (!new_matcher) {
++ mlx5hws_err(ctx, "Rehash error: matcher creation failed\n");
++ return -ENOMEM;
++ }
++
++ ret = mlx5hws_matcher_resize_set_target(old_matcher, new_matcher);
++ if (ret) {
++ mlx5hws_err(ctx, "Rehash error: failed setting resize target\n");
++ mlx5hws_matcher_destroy(new_matcher);
++ return ret;
++ }
++
++ ret = hws_bwc_matcher_move_all(bwc_matcher);
++ if (ret) {
++ mlx5hws_err(ctx, "Rehash error: moving rules failed\n");
++ return -ENOMEM;
++ }
++
++ bwc_matcher->matcher = new_matcher;
++ mlx5hws_matcher_destroy(old_matcher);
++
++ return 0;
++}
++
+ int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher,
+ struct mlx5hws_table *table,
+ u32 priority,
+@@ -210,51 +388,6 @@ int mlx5hws_bwc_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher)
+ return 0;
+ }
+
+-static int hws_bwc_queue_poll(struct mlx5hws_context *ctx,
+- u16 queue_id,
+- u32 *pending_rules,
+- bool drain)
+-{
+- struct mlx5hws_flow_op_result comp[MLX5HWS_BWC_MATCHER_REHASH_BURST_TH];
+- u16 burst_th = hws_bwc_get_burst_th(ctx, queue_id);
+- bool got_comp = *pending_rules >= burst_th;
+- bool queue_full;
+- int err = 0;
+- int ret;
+- int i;
+-
+- /* Check if there are any completions at all */
+- if (!got_comp && !drain)
+- return 0;
+-
+- queue_full = mlx5hws_send_engine_full(&ctx->send_queue[queue_id]);
+- while (queue_full || ((got_comp || drain) && *pending_rules)) {
+- ret = mlx5hws_send_queue_poll(ctx, queue_id, comp, burst_th);
+- if (unlikely(ret < 0)) {
+- mlx5hws_err(ctx, "BWC poll error: polling queue %d returned %d\n",
+- queue_id, ret);
+- return -EINVAL;
+- }
+-
+- if (ret) {
+- (*pending_rules) -= ret;
+- for (i = 0; i < ret; i++) {
+- if (unlikely(comp[i].status != MLX5HWS_FLOW_OP_SUCCESS)) {
+- mlx5hws_err(ctx,
+- "BWC poll error: polling queue %d returned completion with error\n",
+- queue_id);
+- err = -EINVAL;
+- }
+- }
+- queue_full = false;
+- }
+-
+- got_comp = !!ret;
+- }
+-
+- return err;
+-}
+-
+ void
+ mlx5hws_bwc_rule_fill_attr(struct mlx5hws_bwc_matcher *bwc_matcher,
+ u16 bwc_queue_idx,
+@@ -556,138 +689,6 @@ hws_bwc_matcher_find_at(struct mlx5hws_bwc_matcher *bwc_matcher,
+ return -1;
+ }
+
+-static int hws_bwc_matcher_move_all_simple(struct mlx5hws_bwc_matcher *bwc_matcher)
+-{
+- struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
+- u16 bwc_queues = mlx5hws_bwc_queues(ctx);
+- struct mlx5hws_bwc_rule **bwc_rules;
+- struct mlx5hws_rule_attr rule_attr;
+- u32 *pending_rules;
+- int i, j, ret = 0;
+- bool all_done;
+- u16 burst_th;
+-
+- mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr);
+-
+- pending_rules = kcalloc(bwc_queues, sizeof(*pending_rules), GFP_KERNEL);
+- if (!pending_rules)
+- return -ENOMEM;
+-
+- bwc_rules = kcalloc(bwc_queues, sizeof(*bwc_rules), GFP_KERNEL);
+- if (!bwc_rules) {
+- ret = -ENOMEM;
+- goto free_pending_rules;
+- }
+-
+- for (i = 0; i < bwc_queues; i++) {
+- if (list_empty(&bwc_matcher->rules[i]))
+- bwc_rules[i] = NULL;
+- else
+- bwc_rules[i] = list_first_entry(&bwc_matcher->rules[i],
+- struct mlx5hws_bwc_rule,
+- list_node);
+- }
+-
+- do {
+- all_done = true;
+-
+- for (i = 0; i < bwc_queues; i++) {
+- rule_attr.queue_id = mlx5hws_bwc_get_queue_id(ctx, i);
+- burst_th = hws_bwc_get_burst_th(ctx, rule_attr.queue_id);
+-
+- for (j = 0; j < burst_th && bwc_rules[i]; j++) {
+- rule_attr.burst = !!((j + 1) % burst_th);
+- ret = mlx5hws_matcher_resize_rule_move(bwc_matcher->matcher,
+- bwc_rules[i]->rule,
+- &rule_attr);
+- if (unlikely(ret)) {
+- mlx5hws_err(ctx,
+- "Moving BWC rule failed during rehash (%d)\n",
+- ret);
+- goto free_bwc_rules;
+- }
+-
+- all_done = false;
+- pending_rules[i]++;
+- bwc_rules[i] = list_is_last(&bwc_rules[i]->list_node,
+- &bwc_matcher->rules[i]) ?
+- NULL : list_next_entry(bwc_rules[i], list_node);
+-
+- ret = hws_bwc_queue_poll(ctx, rule_attr.queue_id,
+- &pending_rules[i], false);
+- if (unlikely(ret))
+- goto free_bwc_rules;
+- }
+- }
+- } while (!all_done);
+-
+- /* drain all the bwc queues */
+- for (i = 0; i < bwc_queues; i++) {
+- if (pending_rules[i]) {
+- u16 queue_id = mlx5hws_bwc_get_queue_id(ctx, i);
+-
+- mlx5hws_send_engine_flush_queue(&ctx->send_queue[queue_id]);
+- ret = hws_bwc_queue_poll(ctx, queue_id,
+- &pending_rules[i], true);
+- if (unlikely(ret))
+- goto free_bwc_rules;
+- }
+- }
+-
+-free_bwc_rules:
+- kfree(bwc_rules);
+-free_pending_rules:
+- kfree(pending_rules);
+-
+- return ret;
+-}
+-
+-static int hws_bwc_matcher_move_all(struct mlx5hws_bwc_matcher *bwc_matcher)
+-{
+- return hws_bwc_matcher_move_all_simple(bwc_matcher);
+-}
+-
+-static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher)
+-{
+- struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
+- struct mlx5hws_matcher_attr matcher_attr = {0};
+- struct mlx5hws_matcher *old_matcher;
+- struct mlx5hws_matcher *new_matcher;
+- int ret;
+-
+- hws_bwc_matcher_init_attr(&matcher_attr,
+- bwc_matcher->priority,
+- bwc_matcher->size_log);
+-
+- old_matcher = bwc_matcher->matcher;
+- new_matcher = mlx5hws_matcher_create(old_matcher->tbl,
+- &bwc_matcher->mt, 1,
+- bwc_matcher->at,
+- bwc_matcher->num_of_at,
+- &matcher_attr);
+- if (!new_matcher) {
+- mlx5hws_err(ctx, "Rehash error: matcher creation failed\n");
+- return -ENOMEM;
+- }
+-
+- ret = mlx5hws_matcher_resize_set_target(old_matcher, new_matcher);
+- if (ret) {
+- mlx5hws_err(ctx, "Rehash error: failed setting resize target\n");
+- return ret;
+- }
+-
+- ret = hws_bwc_matcher_move_all(bwc_matcher);
+- if (ret) {
+- mlx5hws_err(ctx, "Rehash error: moving rules failed\n");
+- return -ENOMEM;
+- }
+-
+- bwc_matcher->matcher = new_matcher;
+- mlx5hws_matcher_destroy(old_matcher);
+-
+- return 0;
+-}
+-
+ static int
+ hws_bwc_matcher_rehash_size(struct mlx5hws_bwc_matcher *bwc_matcher)
+ {
+diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
+index 95c4a254953a9b..476f7ff0648eaa 100644
+--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
++++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
+@@ -1724,6 +1724,19 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
+ rxbuf_oob = &rxq->rx_oobs[curr];
+ WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
+
++ if (unlikely(pktlen > rxq->datasize)) {
++ /* Increase it even if mana_rx_skb() isn't called. */
++ rxq->rx_cq.work_done++;
++
++ ++ndev->stats.rx_dropped;
++ netdev_warn_once(ndev,
++ "Dropped oversized RX packet: len=%u, datasize=%u\n",
++ pktlen, rxq->datasize);
++
++ /* Reuse the RX buffer since rxbuf_oob is unchanged. */
++ goto drop;
++ }
++
+ mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp);
+
+ /* Unsuccessful refill will have old_buf == NULL.
+diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
+index 279ea0b5695577..55525f45e447b7 100644
+--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
++++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
+@@ -96,6 +96,9 @@ static int nfp_cpp_resource_find(struct nfp_cpp *cpp, struct nfp_resource *res)
+ res->mutex =
+ nfp_cpp_mutex_alloc(cpp,
+ NFP_RESOURCE_TBL_TARGET, addr, key);
++ if (!res->mutex)
++ return -ENOMEM;
++
+ res->cpp_id = NFP_CPP_ID(entry.region.cpp_target,
+ entry.region.cpp_action,
+ entry.region.cpp_token);
+diff --git a/drivers/net/ethernet/realtek/rtase/rtase.h b/drivers/net/ethernet/realtek/rtase/rtase.h
+index b3310e342ccf40..669c12e473fd08 100644
+--- a/drivers/net/ethernet/realtek/rtase/rtase.h
++++ b/drivers/net/ethernet/realtek/rtase/rtase.h
+@@ -186,6 +186,12 @@ enum rtase_sw_flag_content {
+ RTASE_SWF_MSIX_ENABLED = BIT(2),
+ };
+
++enum rtase_parse_result {
++ RTASE_PARSE_OK,
++ RTASE_PARSE_SKIP,
++ RTASE_PARSE_DROP,
++};
++
+ #define RSVD_MASK 0x3FFFC000
+
+ struct rtase_tx_desc {
+@@ -342,4 +348,6 @@ struct rtase_private {
+
+ #define RTASE_MSS_MASK GENMASK(28, 18)
+
++#define RTASE_MIN_PAD_LEN 47
++
+ #endif /* RTASE_H */
+diff --git a/drivers/net/ethernet/realtek/rtase/rtase_main.c b/drivers/net/ethernet/realtek/rtase/rtase_main.c
+index d9d4b7132730f9..a565d5fb6b85c4 100644
+--- a/drivers/net/ethernet/realtek/rtase/rtase_main.c
++++ b/drivers/net/ethernet/realtek/rtase/rtase_main.c
+@@ -61,6 +61,7 @@
+ #include <linux/pci.h>
+ #include <linux/pm_runtime.h>
+ #include <linux/prefetch.h>
++#include <linux/ptp_classify.h>
+ #include <linux/rtnetlink.h>
+ #include <linux/tcp.h>
+ #include <asm/irq.h>
+@@ -1238,6 +1239,199 @@ static u32 rtase_tx_csum(struct sk_buff *skb, const struct net_device *dev)
+ return csum_cmd;
+ }
+
++static enum rtase_parse_result rtase_get_l3_proto(struct sk_buff *skb,
++ __be16 *proto,
++ u32 *network_offset)
++{
++ struct vlan_hdr *vh, _vh;
++ struct ethhdr *eh, _eh;
++ u32 offset = ETH_HLEN;
++
++ eh = skb_header_pointer(skb, 0, sizeof(_eh), &_eh);
++ if (!eh)
++ return RTASE_PARSE_DROP;
++
++ *proto = eh->h_proto;
++
++ while (eth_type_vlan(*proto)) {
++ vh = skb_header_pointer(skb, offset, sizeof(_vh), &_vh);
++ if (!vh)
++ return RTASE_PARSE_DROP;
++
++ *proto = vh->h_vlan_encapsulated_proto;
++ offset += VLAN_HLEN;
++ }
++
++ *network_offset = offset;
++
++ return RTASE_PARSE_OK;
++}
++
++static bool rtase_pad_to_transport_len(struct sk_buff *skb,
++ u32 transport_offset,
++ u32 pad_to_len)
++{
++ u32 trans_data_len;
++ u32 pad_len;
++
++ trans_data_len = skb->len - transport_offset;
++ if (trans_data_len >= pad_to_len)
++ return true;
++
++ if (skb_is_nonlinear(skb)) {
++ if (skb_linearize(skb))
++ return false;
++ }
++
++ pad_len = pad_to_len - trans_data_len;
++ if (__skb_put_padto(skb, skb->len + pad_len, false))
++ return false;
++
++ return true;
++}
++
++static enum rtase_parse_result rtase_get_transport_offset(struct sk_buff *skb,
++ u32 *transport_offset,
++ u8 *transport_proto,
++ u32 *pad_to_len)
++{
++ enum rtase_parse_result ret;
++ struct ipv6hdr *i6h, _i6h;
++ struct iphdr *ih, _ih;
++ bool non_first_frag;
++ __be16 proto;
++ u32 offset;
++ u32 no;
++
++ ret = rtase_get_l3_proto(skb, &proto, &no);
++ if (ret != RTASE_PARSE_OK)
++ return ret;
++
++ switch (proto) {
++ case htons(ETH_P_IP):
++ ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih);
++ if (!ih)
++ return RTASE_PARSE_DROP;
++
++ if (ih->ihl < 5)
++ return RTASE_PARSE_DROP;
++
++ offset = no + ih->ihl * 4;
++ if (offset > skb->len)
++ return RTASE_PARSE_DROP;
++
++ non_first_frag = ntohs(ih->frag_off) & IP_OFFSET;
++
++ if (ih->protocol == IPPROTO_TCP) {
++ if (skb->len - offset < sizeof(struct tcphdr)) {
++ if (non_first_frag) {
++ *transport_offset = offset;
++ *transport_proto = IPPROTO_TCP;
++ *pad_to_len = sizeof(struct tcphdr);
++
++ return RTASE_PARSE_OK;
++ }
++
++ return RTASE_PARSE_DROP;
++ }
++
++ return RTASE_PARSE_SKIP;
++ }
++
++ if (ih->protocol != IPPROTO_UDP)
++ return RTASE_PARSE_SKIP;
++
++ *transport_offset = offset;
++ *transport_proto = IPPROTO_UDP;
++
++ if (skb->len - offset < sizeof(struct udphdr)) {
++ if (non_first_frag) {
++ *pad_to_len = sizeof(struct udphdr);
++
++ return RTASE_PARSE_OK;
++ }
++
++ return RTASE_PARSE_DROP;
++ }
++
++ return RTASE_PARSE_OK;
++
++ case htons(ETH_P_IPV6):
++ i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h);
++ if (!i6h)
++ return RTASE_PARSE_DROP;
++
++ offset = no + sizeof(*i6h);
++
++ if (i6h->nexthdr == IPPROTO_TCP) {
++ if (skb->len - offset < sizeof(struct tcphdr))
++ return RTASE_PARSE_DROP;
++
++ return RTASE_PARSE_SKIP;
++ }
++
++ if (i6h->nexthdr != IPPROTO_UDP)
++ return RTASE_PARSE_SKIP;
++
++ if (skb->len - offset < sizeof(struct udphdr))
++ return RTASE_PARSE_DROP;
++
++ *transport_offset = offset;
++ *transport_proto = IPPROTO_UDP;
++
++ return RTASE_PARSE_OK;
++
++ default:
++ return RTASE_PARSE_SKIP;
++ }
++}
++
++static bool rtase_skb_pad(struct sk_buff *skb)
++{
++ enum rtase_parse_result ret;
++ u32 transport_offset;
++ __be16 *dest, _dest;
++ u32 trans_data_len;
++ u32 pad_to_len = 0;
++ u8 transport_proto;
++ u16 dest_port;
++
++ ret = rtase_get_transport_offset(skb, &transport_offset,
++ &transport_proto, &pad_to_len);
++ if (ret == RTASE_PARSE_SKIP) {
++ return true;
++ } else if (ret == RTASE_PARSE_DROP) {
++ netdev_dbg(skb->dev, "drop malformed packet\n");
++ return false;
++ }
++
++ if (pad_to_len &&
++ !rtase_pad_to_transport_len(skb, transport_offset, pad_to_len))
++ return false;
++
++ if (transport_proto != IPPROTO_UDP)
++ return true;
++
++ trans_data_len = skb->len - transport_offset;
++ if (trans_data_len < offsetof(struct udphdr, len) ||
++ trans_data_len >= RTASE_MIN_PAD_LEN)
++ return true;
++
++ dest = skb_header_pointer(skb,
++ transport_offset +
++ offsetof(struct udphdr, dest),
++ sizeof(_dest), &_dest);
++ if (!dest)
++ return true;
++
++ dest_port = ntohs(*dest);
++ if (dest_port != PTP_EV_PORT && dest_port != PTP_GEN_PORT)
++ return true;
++
++ return rtase_pad_to_transport_len(skb, transport_offset,
++ RTASE_MIN_PAD_LEN);
++}
++
+ static int rtase_xmit_frags(struct rtase_ring *ring, struct sk_buff *skb,
+ u32 opts1, u32 opts2)
+ {
+@@ -1351,6 +1545,9 @@ static netdev_tx_t rtase_start_xmit(struct sk_buff *skb,
+ opts2 |= rtase_tx_csum(skb, dev);
+ }
+
++ if (!rtase_skb_pad(skb))
++ goto err_dma_0;
++
+ frags = rtase_xmit_frags(ring, skb, opts1, opts2);
+ if (unlikely(frags < 0))
+ goto err_dma_0;
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+index 893746b2a49613..44fd0fadc137dd 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+@@ -1032,63 +1032,45 @@ static void stmmac_mac_link_up(struct phylink_config *config,
+ old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
+ ctrl = old_ctrl & ~priv->hw->link.speed_mask;
+
+- if (interface == PHY_INTERFACE_MODE_USXGMII) {
+- switch (speed) {
+- case SPEED_10000:
+- ctrl |= priv->hw->link.xgmii.speed10000;
+- break;
+- case SPEED_5000:
+- ctrl |= priv->hw->link.xgmii.speed5000;
+- break;
+- case SPEED_2500:
++ switch (speed) {
++ case SPEED_100000:
++ ctrl |= priv->hw->link.xlgmii.speed100000;
++ break;
++ case SPEED_50000:
++ ctrl |= priv->hw->link.xlgmii.speed50000;
++ break;
++ case SPEED_40000:
++ ctrl |= priv->hw->link.xlgmii.speed40000;
++ break;
++ case SPEED_25000:
++ ctrl |= priv->hw->link.xlgmii.speed25000;
++ break;
++ case SPEED_10000:
++ ctrl |= priv->hw->link.xgmii.speed10000;
++ break;
++ case SPEED_5000:
++ ctrl |= priv->hw->link.xgmii.speed5000;
++ break;
++ case SPEED_2500:
++ if (interface == PHY_INTERFACE_MODE_USXGMII)
+ ctrl |= priv->hw->link.xgmii.speed2500;
+- break;
+- default:
+- return;
+- }
+- } else if (interface == PHY_INTERFACE_MODE_XLGMII) {
+- switch (speed) {
+- case SPEED_100000:
+- ctrl |= priv->hw->link.xlgmii.speed100000;
+- break;
+- case SPEED_50000:
+- ctrl |= priv->hw->link.xlgmii.speed50000;
+- break;
+- case SPEED_40000:
+- ctrl |= priv->hw->link.xlgmii.speed40000;
+- break;
+- case SPEED_25000:
+- ctrl |= priv->hw->link.xlgmii.speed25000;
+- break;
+- case SPEED_10000:
+- ctrl |= priv->hw->link.xgmii.speed10000;
+- break;
+- case SPEED_2500:
+- ctrl |= priv->hw->link.speed2500;
+- break;
+- case SPEED_1000:
+- ctrl |= priv->hw->link.speed1000;
+- break;
+- default:
+- return;
+- }
+- } else {
+- switch (speed) {
+- case SPEED_2500:
++ else
+ ctrl |= priv->hw->link.speed2500;
+- break;
+- case SPEED_1000:
+- ctrl |= priv->hw->link.speed1000;
+- break;
+- case SPEED_100:
+- ctrl |= priv->hw->link.speed100;
+- break;
+- case SPEED_10:
+- ctrl |= priv->hw->link.speed10;
+- break;
+- default:
+- return;
+- }
++ break;
++ case SPEED_1000:
++ ctrl |= priv->hw->link.speed1000;
++ break;
++ case SPEED_100:
++ ctrl |= priv->hw->link.speed100;
++ break;
++ case SPEED_10:
++ ctrl |= priv->hw->link.speed10;
++ break;
++ default:
++ netdev_err(priv->dev,
++ "unsupported speed %s on %s, leaving the MAC disabled\n",
++ phy_speed_to_str(speed), phy_modes(interface));
++ return;
+ }
+
+ priv->speed = speed;
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
+index 75ad2da1a37f1a..49f133dec810d8 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c
+@@ -446,6 +446,7 @@ static int tc_parse_flow_actions(struct stmmac_priv *priv,
+ }
+
+ #define ETHER_TYPE_FULL_MASK cpu_to_be16(~0)
++#define IP_PROTO_FULL_MASK 0xFF
+
+ static int tc_add_basic_flow(struct stmmac_priv *priv,
+ struct flow_cls_offload *cls,
+@@ -461,6 +462,37 @@ static int tc_add_basic_flow(struct stmmac_priv *priv,
+
+ flow_rule_match_basic(rule, &match);
+
++ /* Both network proto and transport proto not present in the key */
++ if (!match.mask || !(match.mask->n_proto || match.mask->ip_proto)) {
++ NL_SET_ERR_MSG_MOD(cls->common.extack,
++ "filter must specify network or transport protocol");
++ return -EOPNOTSUPP;
++ }
++
++ /* If the proto is present in the key and is not full mask */
++ if ((match.mask->n_proto && match.mask->n_proto != ETHER_TYPE_FULL_MASK) ||
++ (match.mask->ip_proto && match.mask->ip_proto != IP_PROTO_FULL_MASK)) {
++ NL_SET_ERR_MSG_MOD(cls->common.extack,
++ "only full protocol mask is supported");
++ return -EOPNOTSUPP;
++ }
++
++ /* Network proto is present in the key and is not IPv4 */
++ if (match.mask->n_proto && match.key->n_proto != cpu_to_be16(ETH_P_IP)) {
++ NL_SET_ERR_MSG_MOD(cls->common.extack,
++ "only IPv4 network protocol is supported");
++ return -EOPNOTSUPP;
++ }
++
++ /* Transport proto is present in the key and is not TCP or UDP */
++ if (match.mask->ip_proto &&
++ match.key->ip_proto != IPPROTO_TCP &&
++ match.key->ip_proto != IPPROTO_UDP) {
++ NL_SET_ERR_MSG_MOD(cls->common.extack,
++ "only TCP and UDP transport protocols are supported");
++ return -EOPNOTSUPP;
++ }
++
+ entry->ip_proto = match.key->ip_proto;
+ return 0;
+ }
+@@ -598,6 +630,8 @@ static int tc_add_flow(struct stmmac_priv *priv,
+ ret = tc_flow_parsers[i].fn(priv, cls, entry);
+ if (!ret)
+ entry->in_use = true;
++ else if (ret == -EOPNOTSUPP)
++ return ret;
+ }
+
+ if (!entry->in_use)
+@@ -627,6 +661,7 @@ static int tc_del_flow(struct stmmac_priv *priv,
+ entry->in_use = false;
+ entry->cookie = 0;
+ entry->is_l4 = false;
++ entry->action = 0;
+ return ret;
+ }
+
+diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
+index 32fc1bec2728ff..113b2dfa2e4b03 100644
+--- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
++++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
+@@ -744,6 +744,7 @@ static void txgbe_remove(struct pci_dev *pdev)
+
+ netdev = wx->netdev;
+ unregister_netdev(netdev);
++ txgbe_fdir_filter_exit(wx);
+
+ txgbe_remove_phy(txgbe);
+ wx_free_isb_resources(wx);
+diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
+index eea0875e4e5518..3a51f2a72443a1 100644
+--- a/drivers/net/geneve.c
++++ b/drivers/net/geneve.c
+@@ -1694,6 +1694,9 @@ static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
+ struct geneve_config cfg;
+ int err;
+
++ if (!rtnl_dev_link_net_capable(dev, geneve->net))
++ return -EPERM;
++
+ /* If the geneve device is configured for metadata (or externally
+ * controlled, for example, OVS), then nothing can be changed.
+ */
+diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c
+index d7f644dfaa8d9e..b99cab7f10ce7a 100644
+--- a/drivers/net/gtp.c
++++ b/drivers/net/gtp.c
+@@ -665,8 +665,9 @@ static int gtp1u_send_echo_resp(struct gtp_dev *gtp, struct sk_buff *skb)
+ return -1;
+
+ /* pull GTP and UDP headers */
+- skb_pull_data(skb,
+- sizeof(struct gtp1_header_long) + sizeof(struct udphdr));
++ if (!skb_pull_data(skb, sizeof(struct gtp1_header_long) +
++ sizeof(struct udphdr)))
++ return -1;
+
+ gtp_pkt = skb_push(skb, sizeof(struct gtp1u_packet));
+ memset(gtp_pkt, 0, sizeof(struct gtp1u_packet));
+@@ -821,13 +822,17 @@ static int gtp1u_udp_encap_recv(struct gtp_dev *gtp, struct sk_buff *skb)
+ if (!pskb_may_pull(skb, hdrlen))
+ return -1;
+
++ gtp1 = (struct gtp1_header *)(skb->data + sizeof(struct udphdr));
++
++ if (gtp1->flags & GTP1_F_EXTHDR &&
++ gtp_parse_exthdrs(skb, &hdrlen) < 0)
++ return -1;
++
+ if (gtp_inner_proto(skb, hdrlen, &inner_proto) < 0) {
+ netdev_dbg(gtp->dev, "GTP packet does not encapsulate an IP packet\n");
+ return -1;
+ }
+
+- gtp1 = (struct gtp1_header *)(skb->data + sizeof(struct udphdr));
+-
+ pctx = gtp1_pdp_find(gtp, ntohl(gtp1->tid),
+ gtp_proto_to_family(inner_proto));
+ if (!pctx) {
+@@ -835,10 +840,6 @@ static int gtp1u_udp_encap_recv(struct gtp_dev *gtp, struct sk_buff *skb)
+ return 1;
+ }
+
+- if (gtp1->flags & GTP1_F_EXTHDR &&
+- gtp_parse_exthdrs(skb, &hdrlen) < 0)
+- return -1;
+-
+ return gtp_rx(pctx, skb, hdrlen, gtp->role, inner_proto);
+ }
+
+diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c
+index fcaadd111a8a30..0c184f6f91c6f2 100644
+--- a/drivers/net/ipa/ipa_smp2p.c
++++ b/drivers/net/ipa/ipa_smp2p.c
+@@ -234,19 +234,27 @@ ipa_smp2p_init(struct ipa *ipa, struct platform_device *pdev, bool modem_init)
+ &valid_bit);
+ if (IS_ERR(valid_state))
+ return PTR_ERR(valid_state);
+- if (valid_bit >= 32) /* BITS_PER_U32 */
+- return -EINVAL;
++ if (valid_bit >= 32) { /* BITS_PER_U32 */
++ ret = -EINVAL;
++ goto err_valid_state_put;
++ }
+
+ enabled_state = qcom_smem_state_get(dev, "ipa-clock-enabled",
+ &enabled_bit);
+- if (IS_ERR(enabled_state))
+- return PTR_ERR(enabled_state);
+- if (enabled_bit >= 32) /* BITS_PER_U32 */
+- return -EINVAL;
++ if (IS_ERR(enabled_state)) {
++ ret = PTR_ERR(enabled_state);
++ goto err_valid_state_put;
++ }
++ if (enabled_bit >= 32) { /* BITS_PER_U32 */
++ ret = -EINVAL;
++ goto err_enabled_state_put;
++ }
+
+ smp2p = kzalloc(sizeof(*smp2p), GFP_KERNEL);
+- if (!smp2p)
+- return -ENOMEM;
++ if (!smp2p) {
++ ret = -ENOMEM;
++ goto err_enabled_state_put;
++ }
+
+ smp2p->ipa = ipa;
+
+@@ -291,6 +299,10 @@ err_null_smp2p:
+ ipa->smp2p = NULL;
+ mutex_destroy(&smp2p->mutex);
+ kfree(smp2p);
++err_enabled_state_put:
++ qcom_smem_state_put(enabled_state);
++err_valid_state_put:
++ qcom_smem_state_put(valid_state);
+
+ return ret;
+ }
+@@ -307,6 +319,8 @@ void ipa_smp2p_exit(struct ipa *ipa)
+ ipa_smp2p_power_release(ipa);
+ ipa->smp2p = NULL;
+ mutex_destroy(&smp2p->mutex);
++ qcom_smem_state_put(smp2p->enabled_state);
++ qcom_smem_state_put(smp2p->valid_state);
+ kfree(smp2p);
+ }
+
+diff --git a/drivers/net/mctp/mctp-i3c.c b/drivers/net/mctp/mctp-i3c.c
+index 47513ebbc68079..fbcd48f1971a45 100644
+--- a/drivers/net/mctp/mctp-i3c.c
++++ b/drivers/net/mctp/mctp-i3c.c
+@@ -731,18 +731,21 @@ static __init int mctp_i3c_mod_init(void)
+ int rc;
+
+ rc = i3c_register_notifier(&mctp_i3c_notifier);
+- if (rc < 0) {
+- i3c_driver_unregister(&mctp_i3c_driver);
++ if (rc < 0)
+ return rc;
+- }
+
+ i3c_for_each_bus_locked(mctp_i3c_bus_add_new, NULL);
+
+ rc = i3c_driver_register(&mctp_i3c_driver);
+ if (rc < 0)
+- return rc;
++ goto err_unregister_notifier;
+
+ return 0;
++
++err_unregister_notifier:
++ i3c_unregister_notifier(&mctp_i3c_notifier);
++ mctp_i3c_bus_remove_all();
++ return rc;
+ }
+
+ static __exit void mctp_i3c_mod_exit(void)
+diff --git a/drivers/net/mctp/mctp-serial.c b/drivers/net/mctp/mctp-serial.c
+index e63720ec32384b..3312a7bd1de405 100644
+--- a/drivers/net/mctp/mctp-serial.c
++++ b/drivers/net/mctp/mctp-serial.c
+@@ -317,7 +317,7 @@ static void mctp_serial_push_header(struct mctp_serial *dev, u8 c)
+ } else {
+ dev->rxlen = c;
+ dev->rxpos = 0;
+- dev->rxstate = STATE_DATA;
++ dev->rxstate = c > 0 ? STATE_DATA : STATE_TRAILER;
+ dev->rxfcs = crc_ccitt_byte(dev->rxfcs, c);
+ }
+ break;
+diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c
+index 82463f9d50c85c..0152f820c1bbe5 100644
+--- a/drivers/net/pcs/pcs-xpcs.c
++++ b/drivers/net/pcs/pcs-xpcs.c
+@@ -973,6 +973,7 @@ static int xpcs_get_state_c37_sgmii(struct dw_xpcs *xpcs,
+
+ /* Reset link_state */
+ state->link = false;
++ state->an_complete = false;
+ state->speed = SPEED_UNKNOWN;
+ state->duplex = DUPLEX_UNKNOWN;
+ state->pause = 0;
+@@ -984,6 +985,8 @@ static int xpcs_get_state_c37_sgmii(struct dw_xpcs *xpcs,
+ if (ret < 0)
+ return ret;
+
++ state->an_complete = ret & DW_VR_MII_AN_STS_C37_ANCMPLT_INTR;
++
+ if (ret & DW_VR_MII_C37_ANSGM_SP_LNKSTS) {
+ int speed_value;
+
+@@ -1002,34 +1005,13 @@ static int xpcs_get_state_c37_sgmii(struct dw_xpcs *xpcs,
+ state->duplex = DUPLEX_FULL;
+ else
+ state->duplex = DUPLEX_HALF;
+- } else if (ret == DW_VR_MII_AN_STS_C37_ANCMPLT_INTR) {
+- int speed, duplex;
+-
+- state->link = true;
+-
+- speed = xpcs_read(xpcs, MDIO_MMD_VEND2, MDIO_CTRL1);
+- if (speed < 0)
+- return speed;
+-
+- speed &= SGMII_SPEED_SS13 | SGMII_SPEED_SS6;
+- if (speed == SGMII_SPEED_SS6)
+- state->speed = SPEED_1000;
+- else if (speed == SGMII_SPEED_SS13)
+- state->speed = SPEED_100;
+- else if (speed == 0)
+- state->speed = SPEED_10;
+-
+- duplex = xpcs_read(xpcs, MDIO_MMD_VEND2, MII_ADVERTISE);
+- if (duplex < 0)
+- return duplex;
+
+- if (duplex & DW_FULL_DUPLEX)
+- state->duplex = DUPLEX_FULL;
+- else if (duplex & DW_HALF_DUPLEX)
+- state->duplex = DUPLEX_HALF;
++ return 0;
++ }
+
++ /* Clear AN complete status or interrupt */
++ if (state->an_complete)
+ xpcs_write(xpcs, MDIO_MMD_VEND2, DW_VR_MII_AN_INTR_STS, 0);
+- }
+
+ return 0;
+ }
+diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
+index c70994c6a265e3..5d6d69632c847a 100644
+--- a/drivers/net/ppp/ppp_generic.c
++++ b/drivers/net/ppp/ppp_generic.c
+@@ -107,18 +107,6 @@ struct ppp_file {
+ #define PF_TO_PPP(pf) PF_TO_X(pf, struct ppp)
+ #define PF_TO_CHANNEL(pf) PF_TO_X(pf, struct channel)
+
+-/*
+- * Data structure to hold primary network stats for which
+- * we want to use 64 bit storage. Other network stats
+- * are stored in dev->stats of the ppp strucute.
+- */
+-struct ppp_link_stats {
+- u64 rx_packets;
+- u64 tx_packets;
+- u64 rx_bytes;
+- u64 tx_bytes;
+-};
+-
+ /*
+ * Data structure describing one ppp unit.
+ * A ppp unit corresponds to a ppp network interface device
+@@ -162,7 +150,6 @@ struct ppp {
+ struct bpf_prog *active_filter; /* filter for pkts to reset idle */
+ #endif /* CONFIG_PPP_FILTER */
+ struct net *ppp_net; /* the net we belong to */
+- struct ppp_link_stats stats64; /* 64 bit network stats */
+ };
+
+ /*
+@@ -192,6 +179,7 @@ struct channel {
+ struct list_head clist; /* link in list of channels per unit */
+ rwlock_t upl; /* protects `ppp' and 'bridge' */
+ struct channel __rcu *bridge; /* "bridged" ppp channel */
++ struct rcu_head rcu; /* for RCU-deferred free of the channel */
+ #ifdef CONFIG_PPP_MULTILINK
+ u8 avail; /* flag used in multilink stuff */
+ u8 had_frag; /* >= 1 fragments have been sent */
+@@ -825,7 +813,9 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+ case PPPIOCSMRU:
+ if (get_user(val, p))
+ break;
++ ppp_recv_lock(ppp);
+ ppp->mru = val;
++ ppp_recv_unlock(ppp);
+ err = 0;
+ break;
+
+@@ -846,7 +836,9 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+ break;
+
+ case PPPIOCGFLAGS:
++ ppp_lock(ppp);
+ val = ppp->flags | ppp->xstate | ppp->rstate;
++ ppp_unlock(ppp);
+ if (put_user(val, p))
+ break;
+ err = 0;
+@@ -870,7 +862,7 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+ case PPPIOCSDEBUG:
+ if (get_user(val, p))
+ break;
+- ppp->debug = val;
++ WRITE_ONCE(ppp->debug, val);
+ err = 0;
+ break;
+
+@@ -881,16 +873,16 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+ break;
+
+ case PPPIOCGIDLE32:
+- idle32.xmit_idle = (jiffies - ppp->last_xmit) / HZ;
+- idle32.recv_idle = (jiffies - ppp->last_recv) / HZ;
+- if (copy_to_user(argp, &idle32, sizeof(idle32)))
++ idle32.xmit_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_xmit))) / HZ;
++ idle32.recv_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_recv))) / HZ;
++ if (copy_to_user(argp, &idle32, sizeof(idle32)))
+ break;
+ err = 0;
+ break;
+
+ case PPPIOCGIDLE64:
+- idle64.xmit_idle = (jiffies - ppp->last_xmit) / HZ;
+- idle64.recv_idle = (jiffies - ppp->last_recv) / HZ;
++ idle64.xmit_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_xmit))) / HZ;
++ idle64.recv_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_recv))) / HZ;
+ if (copy_to_user(argp, &idle64, sizeof(idle64)))
+ break;
+ err = 0;
+@@ -931,7 +923,7 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+ if (copy_to_user(argp, &npi, sizeof(npi)))
+ break;
+ } else {
+- ppp->npmode[i] = npi.mode;
++ WRITE_ONCE(ppp->npmode[i], npi.mode);
+ /* we may be able to transmit more packets now (??) */
+ netif_wake_queue(ppp->dev);
+ }
+@@ -1466,7 +1458,7 @@ ppp_start_xmit(struct sk_buff *skb, struct net_device *dev)
+ goto outf;
+
+ /* Drop, accept or reject the packet */
+- switch (ppp->npmode[npi]) {
++ switch (READ_ONCE(ppp->npmode[npi])) {
+ case NPMODE_PASS:
+ break;
+ case NPMODE_QUEUE:
+@@ -1544,23 +1536,12 @@ ppp_net_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
+ static void
+ ppp_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64)
+ {
+- struct ppp *ppp = netdev_priv(dev);
+-
+- ppp_recv_lock(ppp);
+- stats64->rx_packets = ppp->stats64.rx_packets;
+- stats64->rx_bytes = ppp->stats64.rx_bytes;
+- ppp_recv_unlock(ppp);
+-
+- ppp_xmit_lock(ppp);
+- stats64->tx_packets = ppp->stats64.tx_packets;
+- stats64->tx_bytes = ppp->stats64.tx_bytes;
+- ppp_xmit_unlock(ppp);
+-
+ stats64->rx_errors = dev->stats.rx_errors;
+ stats64->tx_errors = dev->stats.tx_errors;
+ stats64->rx_dropped = dev->stats.rx_dropped;
+ stats64->tx_dropped = dev->stats.tx_dropped;
+ stats64->rx_length_errors = dev->stats.rx_length_errors;
++ dev_fetch_sw_netstats(stats64, dev->tstats);
+ }
+
+ static int ppp_dev_init(struct net_device *dev)
+@@ -1658,6 +1639,9 @@ static void ppp_setup(struct net_device *dev)
+ dev->type = ARPHRD_PPP;
+ dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
+ dev->priv_destructor = ppp_dev_priv_destructor;
++ dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
++ dev->features = NETIF_F_SG | NETIF_F_FRAGLIST;
++ dev->hw_features = dev->features;
+ netif_keep_dst(dev);
+ }
+
+@@ -1722,6 +1706,10 @@ pad_compress_skb(struct ppp *ppp, struct sk_buff *skb)
+ ppp->xcomp->comp_extra + ppp->dev->hard_header_len;
+ int compressor_skb_size = ppp->dev->mtu +
+ ppp->xcomp->comp_extra + PPP_HDRLEN;
++
++ if (skb_linearize(skb))
++ return NULL;
++
+ new_skb = alloc_skb(new_skb_size, GFP_ATOMIC);
+ if (!new_skb) {
+ if (net_ratelimit())
+@@ -1785,7 +1773,7 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
+ *(__be16 *)skb_push(skb, 2) = htons(PPP_FILTER_OUTBOUND_TAG);
+ if (ppp->pass_filter &&
+ bpf_prog_run(ppp->pass_filter, skb) == 0) {
+- if (ppp->debug & 1)
++ if (READ_ONCE(ppp->debug) & 1)
+ netdev_printk(KERN_DEBUG, ppp->dev,
+ "PPP: outbound frame "
+ "not passed\n");
+@@ -1795,21 +1783,24 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
+ /* if this packet passes the active filter, record the time */
+ if (!(ppp->active_filter &&
+ bpf_prog_run(ppp->active_filter, skb) == 0))
+- ppp->last_xmit = jiffies;
++ WRITE_ONCE(ppp->last_xmit, jiffies);
+ skb_pull(skb, 2);
+ #else
+ /* for data packets, record the time */
+- ppp->last_xmit = jiffies;
++ WRITE_ONCE(ppp->last_xmit, jiffies);
+ #endif /* CONFIG_PPP_FILTER */
+ }
+
+- ++ppp->stats64.tx_packets;
+- ppp->stats64.tx_bytes += skb->len - PPP_PROTO_LEN;
++ dev_sw_netstats_tx_add(ppp->dev, 1, skb->len - PPP_PROTO_LEN);
+
+ switch (proto) {
+ case PPP_IP:
+ if (!ppp->vj || (ppp->flags & SC_COMP_TCP) == 0)
+ break;
++
++ if (skb_linearize(skb))
++ goto drop;
++
+ /* try to do VJ TCP header compression */
+ new_skb = alloc_skb(skb->len + ppp->dev->hard_header_len - 2,
+ GFP_ATOMIC);
+@@ -1907,19 +1898,26 @@ ppp_push(struct ppp *ppp)
+ }
+
+ if ((ppp->flags & SC_MULTILINK) == 0) {
++ struct ppp_channel *chan;
+ /* not doing multilink: send it down the first channel */
+ list = list->next;
+ pch = list_entry(list, struct channel, clist);
+
+ spin_lock(&pch->downl);
+- if (pch->chan) {
+- if (pch->chan->ops->start_xmit(pch->chan, skb))
+- ppp->xmit_pending = NULL;
+- } else {
+- /* channel got unregistered */
++ chan = pch->chan;
++ if (unlikely(!chan || (!chan->direct_xmit && skb_linearize(skb)))) {
++ /* channel got unregistered, or it requires a linear
++ * skb but linearization failed
++ */
+ kfree_skb(skb);
+ ppp->xmit_pending = NULL;
++ goto out;
+ }
++
++ if (chan->ops->start_xmit(chan, skb))
++ ppp->xmit_pending = NULL;
++
++out:
+ spin_unlock(&pch->downl);
+ return;
+ }
+@@ -2004,6 +2002,8 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb)
+ return 0; /* can't take now, leave it in xmit_pending */
+
+ /* Do protocol field compression */
++ if (skb_linearize(skb))
++ goto err_linearize;
+ p = skb->data;
+ len = skb->len;
+ if (*p == 0 && mp_protocol_compress) {
+@@ -2162,7 +2162,8 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb)
+
+ noskb:
+ spin_unlock(&pch->downl);
+- if (ppp->debug & 1)
++ err_linearize:
++ if (READ_ONCE(ppp->debug) & 1)
+ netdev_err(ppp->dev, "PPP: no memory (fragment)\n");
+ ++ppp->dev->stats.tx_errors;
+ ++ppp->nxseq;
+@@ -2482,8 +2483,7 @@ ppp_receive_nonmp_frame(struct ppp *ppp, struct sk_buff *skb)
+ break;
+ }
+
+- ++ppp->stats64.rx_packets;
+- ppp->stats64.rx_bytes += skb->len - 2;
++ dev_sw_netstats_rx_add(ppp->dev, skb->len - PPP_PROTO_LEN);
+
+ npi = proto_to_npindex(proto);
+ if (npi < 0) {
+@@ -2509,7 +2509,7 @@ ppp_receive_nonmp_frame(struct ppp *ppp, struct sk_buff *skb)
+ *(__be16 *)skb_push(skb, 2) = htons(PPP_FILTER_INBOUND_TAG);
+ if (ppp->pass_filter &&
+ bpf_prog_run(ppp->pass_filter, skb) == 0) {
+- if (ppp->debug & 1)
++ if (READ_ONCE(ppp->debug) & 1)
+ netdev_printk(KERN_DEBUG, ppp->dev,
+ "PPP: inbound frame "
+ "not passed\n");
+@@ -2518,14 +2518,14 @@ ppp_receive_nonmp_frame(struct ppp *ppp, struct sk_buff *skb)
+ }
+ if (!(ppp->active_filter &&
+ bpf_prog_run(ppp->active_filter, skb) == 0))
+- ppp->last_recv = jiffies;
++ WRITE_ONCE(ppp->last_recv, jiffies);
+ __skb_pull(skb, 2);
+ } else
+ #endif /* CONFIG_PPP_FILTER */
+- ppp->last_recv = jiffies;
++ WRITE_ONCE(ppp->last_recv, jiffies);
+
+ if ((ppp->dev->flags & IFF_UP) == 0 ||
+- ppp->npmode[npi] != NPMODE_PASS) {
++ READ_ONCE(ppp->npmode[npi]) != NPMODE_PASS) {
+ kfree_skb(skb);
+ } else {
+ /* chop off protocol */
+@@ -2778,7 +2778,7 @@ ppp_mp_reconstruct(struct ppp *ppp)
+ seq = seq_before(minseq, PPP_MP_CB(p)->sequence)?
+ minseq + 1: PPP_MP_CB(p)->sequence;
+
+- if (ppp->debug & 1)
++ if (READ_ONCE(ppp->debug) & 1)
+ netdev_printk(KERN_DEBUG, ppp->dev,
+ "lost frag %u..%u\n",
+ oldseq, seq-1);
+@@ -2827,7 +2827,7 @@ ppp_mp_reconstruct(struct ppp *ppp)
+ struct sk_buff *tmp2;
+
+ skb_queue_reverse_walk_from_safe(list, p, tmp2) {
+- if (ppp->debug & 1)
++ if (READ_ONCE(ppp->debug) & 1)
+ netdev_printk(KERN_DEBUG, ppp->dev,
+ "discarding frag %u\n",
+ PPP_MP_CB(p)->sequence);
+@@ -2849,7 +2849,7 @@ ppp_mp_reconstruct(struct ppp *ppp)
+ skb_queue_walk_safe(list, p, tmp) {
+ if (p == head)
+ break;
+- if (ppp->debug & 1)
++ if (READ_ONCE(ppp->debug) & 1)
+ netdev_printk(KERN_DEBUG, ppp->dev,
+ "discarding frag %u\n",
+ PPP_MP_CB(p)->sequence);
+@@ -2857,7 +2857,7 @@ ppp_mp_reconstruct(struct ppp *ppp)
+ kfree_skb(p);
+ }
+
+- if (ppp->debug & 1)
++ if (READ_ONCE(ppp->debug) & 1)
+ netdev_printk(KERN_DEBUG, ppp->dev,
+ " missed pkts %u..%u\n",
+ ppp->nextseq,
+@@ -3167,7 +3167,8 @@ ppp_ccp_peek(struct ppp *ppp, struct sk_buff *skb, int inbound)
+ if (!ppp->rc_state)
+ break;
+ if (ppp->rcomp->decomp_init(ppp->rc_state, dp, len,
+- ppp->file.index, 0, ppp->mru, ppp->debug)) {
++ ppp->file.index, 0, ppp->mru,
++ READ_ONCE(ppp->debug))) {
+ ppp->rstate |= SC_DECOMP_RUN;
+ ppp->rstate &= ~(SC_DC_ERROR | SC_DC_FERROR);
+ }
+@@ -3176,7 +3177,8 @@ ppp_ccp_peek(struct ppp *ppp, struct sk_buff *skb, int inbound)
+ if (!ppp->xc_state)
+ break;
+ if (ppp->xcomp->comp_init(ppp->xc_state, dp, len,
+- ppp->file.index, 0, ppp->debug))
++ ppp->file.index, 0,
++ READ_ONCE(ppp->debug)))
+ ppp->xstate |= SC_COMP_RUN;
+ }
+ break;
+@@ -3311,14 +3313,25 @@ static void
+ ppp_get_stats(struct ppp *ppp, struct ppp_stats *st)
+ {
+ struct slcompress *vj = ppp->vj;
++ int cpu;
+
+ memset(st, 0, sizeof(*st));
+- st->p.ppp_ipackets = ppp->stats64.rx_packets;
++ for_each_possible_cpu(cpu) {
++ struct pcpu_sw_netstats *p = per_cpu_ptr(ppp->dev->tstats, cpu);
++ u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
++
++ rx_packets = u64_stats_read(&p->rx_packets);
++ rx_bytes = u64_stats_read(&p->rx_bytes);
++ tx_packets = u64_stats_read(&p->tx_packets);
++ tx_bytes = u64_stats_read(&p->tx_bytes);
++
++ st->p.ppp_ipackets += rx_packets;
++ st->p.ppp_ibytes += rx_bytes;
++ st->p.ppp_opackets += tx_packets;
++ st->p.ppp_obytes += tx_bytes;
++ }
+ st->p.ppp_ierrors = ppp->dev->stats.rx_errors;
+- st->p.ppp_ibytes = ppp->stats64.rx_bytes;
+- st->p.ppp_opackets = ppp->stats64.tx_packets;
+ st->p.ppp_oerrors = ppp->dev->stats.tx_errors;
+- st->p.ppp_obytes = ppp->stats64.tx_bytes;
+ if (!vj)
+ return;
+ st->vj.vjs_packets = vj->sls_o_compressed + vj->sls_o_uncompressed;
+@@ -3507,6 +3520,10 @@ ppp_connect_channel(struct channel *pch, int unit)
+ ret = -ENOTCONN;
+ goto outl;
+ }
++ if (pch->chan->direct_xmit)
++ ppp->dev->priv_flags |= IFF_NO_QUEUE;
++ else
++ ppp->dev->priv_flags &= ~IFF_NO_QUEUE;
+ spin_unlock_bh(&pch->downl);
+ if (pch->file.hdrlen > ppp->file.hdrlen)
+ ppp->file.hdrlen = pch->file.hdrlen;
+@@ -3555,6 +3572,18 @@ ppp_disconnect_channel(struct channel *pch)
+ return err;
+ }
+
++/* Purge after the grace period: a late ppp_input() may still queue an
++ * skb on pch->file.rq before the last RCU reader drains.
++ */
++static void ppp_release_channel_free(struct rcu_head *rcu)
++{
++ struct channel *pch = container_of(rcu, struct channel, rcu);
++
++ skb_queue_purge(&pch->file.xq);
++ skb_queue_purge(&pch->file.rq);
++ kfree(pch);
++}
++
+ /*
+ * Free up the resources used by a ppp channel.
+ */
+@@ -3570,9 +3599,7 @@ static void ppp_destroy_channel(struct channel *pch)
+ pr_err("ppp: destroying undead channel %p !\n", pch);
+ return;
+ }
+- skb_queue_purge(&pch->file.xq);
+- skb_queue_purge(&pch->file.rq);
+- kfree(pch);
++ call_rcu(&pch->rcu, ppp_release_channel_free);
+ }
+
+ static void __exit ppp_cleanup(void)
+@@ -3585,6 +3612,7 @@ static void __exit ppp_cleanup(void)
+ device_destroy(&ppp_class, MKDEV(PPP_MAJOR, 0));
+ class_unregister(&ppp_class);
+ unregister_pernet_device(&ppp_net_ops);
++ rcu_barrier(); /* wait for RCU callbacks before module unload */
+ }
+
+ /*
+diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c
+index 937cf9b17f9aeb..8ff2024dde6044 100644
+--- a/drivers/net/ppp/pppoe.c
++++ b/drivers/net/ppp/pppoe.c
+@@ -699,6 +699,7 @@ static int pppoe_connect(struct socket *sock, struct sockaddr *uservaddr,
+ po->chan.mtu = dev->mtu - sizeof(struct pppoe_hdr) - 2;
+ po->chan.private = sk;
+ po->chan.ops = &pppoe_chan_ops;
++ po->chan.direct_xmit = true;
+
+ error = ppp_register_net_channel(dev_net(dev), &po->chan);
+ if (error) {
+@@ -899,6 +900,7 @@ static int pppoe_sendmsg(struct socket *sock, struct msghdr *m,
+ dev_hard_header(skb, dev, ETH_P_PPP_SES,
+ po->pppoe_pa.remote, NULL, total_len);
+
++ ph = pppoe_hdr(skb);
+ memcpy(ph, &hdr, sizeof(struct pppoe_hdr));
+
+ ph->length = htons(total_len);
+diff --git a/drivers/net/ppp/pptp.c b/drivers/net/ppp/pptp.c
+index cec3bb22471b54..90737cb718928a 100644
+--- a/drivers/net/ppp/pptp.c
++++ b/drivers/net/ppp/pptp.c
+@@ -469,6 +469,7 @@ static int pptp_connect(struct socket *sock, struct sockaddr *uservaddr,
+ po->chan.mtu -= PPTP_HEADER_OVERHEAD;
+
+ po->chan.hdrlen = 2 + sizeof(struct pptp_gre_header);
++ po->chan.direct_xmit = true;
+ error = ppp_register_channel(&po->chan);
+ if (error) {
+ pr_err("PPTP: failed to register PPP channel (%d)\n", error);
+diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c
+index fb362ee248ff26..e8006c94cedc50 100644
+--- a/drivers/net/slip/slip.c
++++ b/drivers/net/slip/slip.c
+@@ -693,6 +693,8 @@ static void slip_receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp,
+ if (!sl || sl->magic != SLIP_MAGIC || !netif_running(sl->dev))
+ return;
+
++ spin_lock_bh(&sl->lock);
++
+ /* Read the characters out of the buffer */
+ while (count--) {
+ if (fp && *fp++) {
+@@ -708,6 +710,8 @@ static void slip_receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp,
+ #endif
+ slip_unesc(sl, *cp++);
+ }
++
++ spin_unlock_bh(&sl->lock);
+ }
+
+ /************************************
+diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
+index e1e472b3a301ef..538738231a5a95 100644
+--- a/drivers/net/vmxnet3/vmxnet3_drv.c
++++ b/drivers/net/vmxnet3/vmxnet3_drv.c
+@@ -1518,7 +1518,11 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
+ struct ipv6hdr *ipv6;
+ struct tcphdr *tcp;
+ } hdr;
+- BUG_ON(gdesc->rcd.tcp == 0);
++
++ /* v4/v6/tcp then describe the inner header, which we can't locate. */
++ if ((le32_to_cpu(gdesc->dword[0]) & (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)) ||
++ gdesc->rcd.tcp == 0)
++ return 0;
+
+ maplen = skb_headlen(skb);
+ if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen))
+@@ -1532,15 +1536,21 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
+
+ hdr.eth = eth_hdr(skb);
+ if (gdesc->rcd.v4) {
+- BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) &&
+- hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP));
++ if (hdr.eth->h_proto != htons(ETH_P_IP) &&
++ hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP))
++ return 0;
++
+ hdr.ptr += hlen;
+- BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
++ if (hdr.ipv4->protocol != IPPROTO_TCP)
++ return 0;
++
+ hlen = hdr.ipv4->ihl << 2;
+ hdr.ptr += hdr.ipv4->ihl << 2;
+ } else if (gdesc->rcd.v6) {
+- BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) &&
+- hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6));
++ if (hdr.eth->h_proto != htons(ETH_P_IPV6) &&
++ hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6))
++ return 0;
++
+ hdr.ptr += hlen;
+ /* Use an estimated value, since we also need to handle
+ * TSO case.
+diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
+index 596efc51cac77b..c6d80973fd9627 100644
+--- a/drivers/net/vxlan/vxlan_core.c
++++ b/drivers/net/vxlan/vxlan_core.c
+@@ -4381,6 +4381,9 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],
+ struct vxlan_rdst *dst;
+ int err;
+
++ if (!rtnl_dev_link_net_capable(dev, vxlan->net))
++ return -EPERM;
++
+ dst = &vxlan->default_dst;
+ err = vxlan_nl2conf(tb, data, dev, &conf, true, extack);
+ if (err)
+diff --git a/drivers/net/vxlan/vxlan_mdb.c b/drivers/net/vxlan/vxlan_mdb.c
+index ec86d1c024834d..165b9f4a8eef6a 100644
+--- a/drivers/net/vxlan/vxlan_mdb.c
++++ b/drivers/net/vxlan/vxlan_mdb.c
+@@ -42,6 +42,7 @@ struct vxlan_mdb_remote {
+ };
+
+ #define VXLAN_SGRP_F_DELETE BIT(0)
++#define VXLAN_SGRP_F_NEW BIT(1)
+
+ struct vxlan_mdb_src_entry {
+ struct hlist_node node;
+@@ -844,6 +845,7 @@ vxlan_mdb_remote_src_add(const struct vxlan_mdb_config *cfg,
+ ent = vxlan_mdb_remote_src_entry_add(remote, &src->addr);
+ if (!ent)
+ return -ENOMEM;
++ ent->flags |= VXLAN_SGRP_F_NEW;
+ } else if (!(cfg->nlflags & NLM_F_REPLACE)) {
+ NL_SET_ERR_MSG_MOD(extack, "Source entry already exists");
+ return -EEXIST;
+@@ -853,15 +855,16 @@ vxlan_mdb_remote_src_add(const struct vxlan_mdb_config *cfg,
+ if (err)
+ goto err_src_del;
+
+- /* Clear flags in case source entry was marked for deletion as part of
+- * replace flow.
++ /* Clear the deletion mark so the entry survives the replace sweep.
++ * The new mark is retained until the whole operation succeeds.
+ */
+- ent->flags = 0;
++ ent->flags &= ~VXLAN_SGRP_F_DELETE;
+
+ return 0;
+
+ err_src_del:
+- vxlan_mdb_remote_src_entry_del(ent);
++ if (ent->flags & VXLAN_SGRP_F_NEW)
++ vxlan_mdb_remote_src_entry_del(ent);
+ return err;
+ }
+
+@@ -889,11 +892,19 @@ static int vxlan_mdb_remote_srcs_add(const struct vxlan_mdb_config *cfg,
+ goto err_src_del;
+ }
+
++ hlist_for_each_entry(ent, &remote->src_list, node)
++ ent->flags &= ~VXLAN_SGRP_F_NEW;
++
+ return 0;
+
+ err_src_del:
+- hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node)
+- vxlan_mdb_remote_src_del(cfg->vxlan, &cfg->group, remote, ent);
++ hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node) {
++ if (ent->flags & VXLAN_SGRP_F_NEW)
++ vxlan_mdb_remote_src_del(cfg->vxlan, &cfg->group, remote,
++ ent);
++ else
++ ent->flags &= ~VXLAN_SGRP_F_DELETE;
++ }
+ return err;
+ }
+
+@@ -1069,7 +1080,7 @@ vxlan_mdb_remote_srcs_replace(const struct vxlan_mdb_config *cfg,
+
+ err = vxlan_mdb_remote_srcs_add(cfg, remote, extack);
+ if (err)
+- goto err_clear_delete;
++ return err;
+
+ hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node) {
+ if (ent->flags & VXLAN_SGRP_F_DELETE)
+@@ -1078,11 +1089,6 @@ vxlan_mdb_remote_srcs_replace(const struct vxlan_mdb_config *cfg,
+ }
+
+ return 0;
+-
+-err_clear_delete:
+- hlist_for_each_entry(ent, &remote->src_list, node)
+- ent->flags &= ~VXLAN_SGRP_F_DELETE;
+- return err;
+ }
+
+ static int vxlan_mdb_remote_replace(const struct vxlan_mdb_config *cfg,
+diff --git a/drivers/net/wan/wanxl.c b/drivers/net/wan/wanxl.c
+index 5a9e262188efe1..c38dd741401e13 100644
+--- a/drivers/net/wan/wanxl.c
++++ b/drivers/net/wan/wanxl.c
+@@ -514,7 +514,8 @@ static void wanxl_pci_remove_one(struct pci_dev *pdev)
+ if (card->irq)
+ free_irq(card->irq, card);
+
+- wanxl_reset(card);
++ if (card->plx)
++ wanxl_reset(card);
+
+ for (i = 0; i < RX_QUEUE_LENGTH; i++)
+ if (card->rx_skbs[i]) {
+diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
+index 78b0fa8d4f3391..6cc252d00ef130 100644
+--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
++++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
+@@ -4614,6 +4614,9 @@ static void ath11k_hal_rx_msdu_list_get(struct ath11k *ar,
+ msdu_details = &msdu_link->msdu_link[0];
+
+ for (i = 0; i < HAL_RX_NUM_MSDU_DESC; i++) {
++ if (!i && FIELD_GET(BUFFER_ADDR_INFO0_ADDR,
++ msdu_details[i].buf_addr_info.info0) == 0)
++ break;
+ if (FIELD_GET(BUFFER_ADDR_INFO0_ADDR,
+ msdu_details[i].buf_addr_info.info0) == 0) {
+ msdu_desc_info = &msdu_details[i - 1].rx_msdu_info;
+diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c
+index eee83eb6b2c3c0..6b0e52a6fb89b6 100644
+--- a/drivers/net/wireless/ath/ath11k/pci.c
++++ b/drivers/net/wireless/ath/ath11k/pci.c
+@@ -197,6 +197,8 @@ static void ath11k_pci_soc_global_reset(struct ath11k_base *ab)
+ val |= PCIE_SOC_GLOBAL_RESET_V;
+
+ ath11k_pcic_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
++ /* Flush the posted write to the device */
++ ath11k_pcic_read32(ab, PCIE_SOC_GLOBAL_RESET);
+
+ /* TODO: exact time to sleep is uncertain */
+ delay = 10;
+@@ -206,6 +208,8 @@ static void ath11k_pci_soc_global_reset(struct ath11k_base *ab)
+ val &= ~PCIE_SOC_GLOBAL_RESET_V;
+
+ ath11k_pcic_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
++ /* Flush the posted write to the device */
++ ath11k_pcic_read32(ab, PCIE_SOC_GLOBAL_RESET);
+
+ mdelay(delay);
+
+diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c
+index 3ffb7723b67318..0c2527d85532fa 100644
+--- a/drivers/net/wireless/ath/ath11k/qmi.c
++++ b/drivers/net/wireless/ath/ath11k/qmi.c
+@@ -3295,9 +3295,14 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work)
+ clear_bit(ATH11K_FLAG_CRASH_FLUSH,
+ &ab->dev_flags);
+ clear_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags);
+- ath11k_core_qmi_firmware_ready(ab);
+- set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags);
+-
++ if (!test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) {
++ ret = ath11k_core_qmi_firmware_ready(ab);
++ if (ret) {
++ set_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags);
++ break;
++ }
++ set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags);
++ }
+ break;
+ case ATH11K_QMI_EVENT_COLD_BOOT_CAL_DONE:
+ break;
+diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c
+index 0ac92a606cea0c..4ba0c1bb32341b 100644
+--- a/drivers/net/wireless/ath/ath12k/pci.c
++++ b/drivers/net/wireless/ath/ath12k/pci.c
+@@ -225,6 +225,8 @@ static void ath12k_pci_soc_global_reset(struct ath12k_base *ab)
+ val |= PCIE_SOC_GLOBAL_RESET_V;
+
+ ath12k_pci_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
++ /* Flush the posted write to the device */
++ ath12k_pci_read32(ab, PCIE_SOC_GLOBAL_RESET);
+
+ /* TODO: exact time to sleep is uncertain */
+ delay = 10;
+@@ -234,6 +236,8 @@ static void ath12k_pci_soc_global_reset(struct ath12k_base *ab)
+ val &= ~PCIE_SOC_GLOBAL_RESET_V;
+
+ ath12k_pci_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
++ /* Flush the posted write to the device */
++ ath12k_pci_read32(ab, PCIE_SOC_GLOBAL_RESET);
+
+ mdelay(delay);
+
+diff --git a/drivers/net/wireless/ath/ath6kl/txrx.c b/drivers/net/wireless/ath/ath6kl/txrx.c
+index 80e66acc5cf6e9..a39c8150587eee 100644
+--- a/drivers/net/wireless/ath/ath6kl/txrx.c
++++ b/drivers/net/wireless/ath/ath6kl/txrx.c
+@@ -1722,13 +1722,15 @@ void aggr_recv_addba_req_evt(struct ath6kl_vif *vif, u8 tid_mux, u16 seq_no,
+
+ rxtid = &aggr_conn->rx_tid[tid];
+
+- if (win_sz < AGGR_WIN_SZ_MIN || win_sz > AGGR_WIN_SZ_MAX)
+- ath6kl_dbg(ATH6KL_DBG_WLAN_RX, "%s: win_sz %d, tid %d\n",
+- __func__, win_sz, tid);
+-
+ if (rxtid->aggr)
+ aggr_delete_tid_state(aggr_conn, tid);
+
++ if (win_sz < AGGR_WIN_SZ_MIN || win_sz > AGGR_WIN_SZ_MAX) {
++ ath6kl_dbg(ATH6KL_DBG_WLAN_RX, "%s: win_sz %d, tid %d\n",
++ __func__, win_sz, tid);
++ return;
++ }
++
+ rxtid->seq_next = seq_no;
+ hold_q_size = TID_WINDOW_SZ(win_sz) * sizeof(struct skb_hold_q);
+ rxtid->hold_q = kzalloc(hold_q_size, GFP_KERNEL);
+diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c
+index 3787b9fb007559..447896b871c3e2 100644
+--- a/drivers/net/wireless/ath/ath6kl/wmi.c
++++ b/drivers/net/wireless/ath/ath6kl/wmi.c
+@@ -484,6 +484,18 @@ static int ath6kl_wmi_tx_complete_event_rx(u8 *datap, int len)
+
+ evt = (struct wmi_tx_complete_event *) datap;
+
++ if (len < sizeof(*evt)) {
++ ath6kl_dbg(ATH6KL_DBG_WMI, "tx complete: invalid len %d\n",
++ len);
++ return -EINVAL;
++ }
++
++ if (len < sizeof(*evt) + evt->num_msg * sizeof(struct tx_complete_msg_v1)) {
++ ath6kl_dbg(ATH6KL_DBG_WMI, "tx complete: invalid len %d for %u msgs\n",
++ len, evt->num_msg);
++ return -EINVAL;
++ }
++
+ ath6kl_dbg(ATH6KL_DBG_WMI, "comp: %d %d %d\n",
+ evt->num_msg, evt->msg_len, evt->msg_type);
+
+@@ -862,6 +874,14 @@ static int ath6kl_wmi_connect_event_rx(struct wmi *wmi, u8 *datap, int len,
+
+ ev = (struct wmi_connect_event *) datap;
+
++ if (len < sizeof(*ev) + ev->beacon_ie_len +
++ ev->assoc_req_len + ev->assoc_resp_len) {
++ ath6kl_dbg(ATH6KL_DBG_WMI,
++ "connect event: IE lengths %u+%u+%u exceed buffer %d\n",
++ ev->beacon_ie_len, ev->assoc_req_len,
++ ev->assoc_resp_len, len);
++ return -EINVAL;
++ }
+ if (vif->nw_type == AP_NETWORK) {
+ /* AP mode start/STA connected event */
+ struct net_device *dev = vif->ndev;
+diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
+index 7265766cddbdeb..dbf94333664ec5 100644
+--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
++++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
+@@ -1225,15 +1225,10 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev,
+ ret = request_firmware_nowait(THIS_MODULE, true, hif_dev->fw_name,
+ &hif_dev->udev->dev, GFP_KERNEL,
+ hif_dev, ath9k_hif_usb_firmware_cb);
+- if (ret) {
++ if (ret)
+ dev_err(&hif_dev->udev->dev,
+ "ath9k_htc: Async request for firmware %s failed\n",
+ hif_dev->fw_name);
+- return ret;
+- }
+-
+- dev_info(&hif_dev->udev->dev, "ath9k_htc: Firmware %s requested\n",
+- hif_dev->fw_name);
+
+ return ret;
+ }
+diff --git a/drivers/net/wireless/ath/carl9170/rx.c b/drivers/net/wireless/ath/carl9170/rx.c
+index 908c4c8b7f8256..bda30b1f940459 100644
+--- a/drivers/net/wireless/ath/carl9170/rx.c
++++ b/drivers/net/wireless/ath/carl9170/rx.c
+@@ -150,7 +150,8 @@ static void carl9170_cmd_callback(struct ar9170 *ar, u32 len, void *buffer)
+ spin_lock(&ar->cmd_lock);
+ if (ar->readbuf) {
+ if (len >= 4)
+- memcpy(ar->readbuf, buffer + 4, len - 4);
++ memcpy(ar->readbuf, buffer + 4,
++ min_t(u32, len - 4, ar->readlen));
+
+ ar->readbuf = NULL;
+ }
+@@ -917,7 +918,9 @@ static void carl9170_rx_stream(struct ar9170 *ar, void *buf, unsigned int len)
+ }
+ }
+
+- skb_put_data(ar->rx_failover, tbuf, tlen);
++ skb_put_data(ar->rx_failover, tbuf,
++ min_t(unsigned int, tlen,
++ ar->rx_failover_missing));
+ ar->rx_failover_missing -= tlen;
+
+ if (ar->rx_failover_missing <= 0) {
+diff --git a/drivers/net/wireless/ath/carl9170/tx.c b/drivers/net/wireless/ath/carl9170/tx.c
+index 0226c31a6cae26..8280d264d5e2e8 100644
+--- a/drivers/net/wireless/ath/carl9170/tx.c
++++ b/drivers/net/wireless/ath/carl9170/tx.c
+@@ -693,7 +693,7 @@ void carl9170_tx_process_status(struct ar9170 *ar,
+ unsigned int i;
+
+ for (i = 0; i < cmd->hdr.ext; i++) {
+- if (WARN_ON(i > ((cmd->hdr.len / 2) + 1))) {
++ if (WARN_ON(i >= (cmd->hdr.len / 2))) {
+ print_hex_dump_bytes("UU:", DUMP_PREFIX_NONE,
+ (void *) cmd, cmd->hdr.len + 4);
+ break;
+diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
+index e8f1d30a8d73c5..a1a0a9223e74ac 100644
+--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
++++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
+@@ -1493,6 +1493,7 @@ out:
+ }
+
+ static int wil_cfg80211_set_channel(struct wiphy *wiphy,
++ struct net_device *dev,
+ struct cfg80211_chan_def *chandef)
+ {
+ struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+diff --git a/drivers/net/wireless/atmel/at76c50x-usb.c b/drivers/net/wireless/atmel/at76c50x-usb.c
+index 97ea7ab0f49102..b4850df4c98469 100644
+--- a/drivers/net/wireless/atmel/at76c50x-usb.c
++++ b/drivers/net/wireless/atmel/at76c50x-usb.c
+@@ -1521,13 +1521,16 @@ static inline int at76_guess_freq(struct at76_priv *priv)
+
+ if (ieee80211_is_probe_resp(hdr->frame_control)) {
+ el_off = offsetof(struct ieee80211_mgmt, u.probe_resp.variable);
+- el = ((struct ieee80211_mgmt *)hdr)->u.probe_resp.variable;
+ } else if (ieee80211_is_beacon(hdr->frame_control)) {
+ el_off = offsetof(struct ieee80211_mgmt, u.beacon.variable);
+- el = ((struct ieee80211_mgmt *)hdr)->u.beacon.variable;
+ } else {
+ goto exit;
+ }
++
++ if (len < el_off)
++ goto exit;
++
++ el = priv->rx_skb->data + el_off;
+ len -= el_off;
+
+ el = cfg80211_find_ie(WLAN_EID_DS_PARAMS, el, len);
+diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+index 2b8a85a7bf9eba..c3a3a081791095 100644
+--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+@@ -2122,7 +2122,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme)
+ sme->crypto.akm_suites[0]);
+ return -EINVAL;
+ }
+- } else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED)) {
++ } else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_1X_SHA256)) {
+ switch (sme->crypto.akm_suites[0]) {
+ case WLAN_AKM_SUITE_8021X:
+ val = WPA2_AUTH_UNSPECIFIED;
+diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+index 5dee54819fbdfb..c53099a3e25035 100644
+--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+@@ -1376,16 +1376,20 @@ fail:
+ static void
+ brcmf_pcie_release_scratchbuffers(struct brcmf_pciedev_info *devinfo)
+ {
+- if (devinfo->shared.scratch)
++ if (devinfo->shared.scratch) {
+ dma_free_coherent(&devinfo->pdev->dev,
+ BRCMF_DMA_D2H_SCRATCH_BUF_LEN,
+ devinfo->shared.scratch,
+ devinfo->shared.scratch_dmahandle);
+- if (devinfo->shared.ringupd)
++ devinfo->shared.scratch = NULL;
++ }
++ if (devinfo->shared.ringupd) {
+ dma_free_coherent(&devinfo->pdev->dev,
+ BRCMF_DMA_D2H_RINGUPD_BUF_LEN,
+ devinfo->shared.ringupd,
+ devinfo->shared.ringupd_dmahandle);
++ devinfo->shared.ringupd = NULL;
++ }
+ }
+
+ static int brcmf_pcie_init_scratchbuffers(struct brcmf_pciedev_info *devinfo)
+diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+index 71bb8b699731ba..a1b26144607698 100644
+--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+@@ -4458,6 +4458,7 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
+ bus->sdiodev = sdiodev;
+ sdiodev->bus = bus;
+ skb_queue_head_init(&bus->glom);
++ INIT_WORK(&bus->datawork, brcmf_sdio_dataworker);
+ bus->txbound = BRCMF_TXBOUND;
+ bus->rxbound = BRCMF_RXBOUND;
+ bus->txminmax = BRCMF_TXMINMAX;
+@@ -4471,7 +4472,6 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
+ goto fail;
+ }
+ brcmf_sdiod_freezer_count(sdiodev);
+- INIT_WORK(&bus->datawork, brcmf_sdio_dataworker);
+ bus->brcmf_wq = wq;
+
+ /* attempt to attach to the dongle */
+diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+index fe75941c584d15..95778dfdfa6d81 100644
+--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c
++++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+@@ -6167,6 +6167,8 @@ static int ipw2100_pci_init_one(struct pci_dev *pci_dev,
+ if (err) {
+ printk(KERN_WARNING DRV_NAME
+ "Error calling pci_enable_device.\n");
++ free_libipw(dev, 0);
++ pci_iounmap(pci_dev, ioaddr);
+ return err;
+ }
+
+@@ -6179,16 +6181,14 @@ static int ipw2100_pci_init_one(struct pci_dev *pci_dev,
+ if (err) {
+ printk(KERN_WARNING DRV_NAME
+ "Error calling pci_set_dma_mask.\n");
+- pci_disable_device(pci_dev);
+- return err;
++ goto fail;
+ }
+
+ err = pci_request_regions(pci_dev, DRV_NAME);
+ if (err) {
+ printk(KERN_WARNING DRV_NAME
+ "Error calling pci_request_regions.\n");
+- pci_disable_device(pci_dev);
+- return err;
++ goto fail;
+ }
+
+ /* We disable the RETRY_TIMEOUT register (0x41) to keep
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+index 817ddc7103af27..dd31b9b2cd39bf 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+@@ -3144,7 +3144,7 @@ static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm,
+ struct iwl_wowlan_status_data *status,
+ u32 len)
+ {
+- u32 data_size, packet_len = le32_to_cpu(notif->wake_packet_length);
++ u32 data_size, packet_len;
+
+ if (len < sizeof(*notif)) {
+ IWL_ERR(mvm, "Invalid WoWLAN wake packet notification!\n");
+@@ -3163,6 +3163,7 @@ static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm,
+ return -EIO;
+ }
+
++ packet_len = le32_to_cpu(notif->wake_packet_length);
+ data_size = len - offsetof(struct iwl_wowlan_wake_pkt_notif, wake_packet);
+
+ /* data_size got the padding from the notification, remove it. */
+diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+index 2b9a684cf61d57..4bad012472e1e9 100644
+--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
++++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+@@ -976,12 +976,22 @@ int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm)
+ return ret;
+ }
+
++ if (IWL_FW_CHECK(mvm,
++ iwl_rx_packet_payload_len(cmd.resp_pkt) !=
++ sizeof(*resp),
++ "Wrong size for iwl_geo_tx_power_profiles_resp: %d\n",
++ iwl_rx_packet_payload_len(cmd.resp_pkt))) {
++ ret = -EIO;
++ goto out;
++ }
++
+ resp = (void *)cmd.resp_pkt->data;
+ ret = le32_to_cpu(resp->profile_idx);
+
+ if (WARN_ON(ret > BIOS_GEO_MAX_PROFILE_NUM))
+ ret = -EIO;
+
++out:
+ iwl_free_resp(&cmd);
+ return ret;
+ }
+diff --git a/drivers/net/wireless/intersil/p54/txrx.c b/drivers/net/wireless/intersil/p54/txrx.c
+index 2deb1bb54f24bd..dc7bf54e30bba1 100644
+--- a/drivers/net/wireless/intersil/p54/txrx.c
++++ b/drivers/net/wireless/intersil/p54/txrx.c
+@@ -499,11 +499,19 @@ static void p54_rx_eeprom_readback(struct p54_common *priv,
+ if (le16_to_cpu(eeprom->v2.len) != priv->eeprom_slice_size)
+ return;
+
++ if (eeprom->v2.data + priv->eeprom_slice_size >
++ skb_tail_pointer(skb))
++ return;
++
+ memcpy(priv->eeprom, eeprom->v2.data, priv->eeprom_slice_size);
+ } else {
+ if (le16_to_cpu(eeprom->v1.len) != priv->eeprom_slice_size)
+ return;
+
++ if (eeprom->v1.data + priv->eeprom_slice_size >
++ skb_tail_pointer(skb))
++ return;
++
+ memcpy(priv->eeprom, eeprom->v1.data, priv->eeprom_slice_size);
+ }
+
+diff --git a/drivers/net/wireless/marvell/libertas/cfg.c b/drivers/net/wireless/marvell/libertas/cfg.c
+index 37bb788f83e36e..309556541a83ed 100644
+--- a/drivers/net/wireless/marvell/libertas/cfg.c
++++ b/drivers/net/wireless/marvell/libertas/cfg.c
+@@ -486,6 +486,7 @@ static int lbs_add_wps_enrollee_tlv(u8 *tlv, const u8 *ie, size_t ie_len)
+ */
+
+ static int lbs_cfg_set_monitor_channel(struct wiphy *wiphy,
++ struct net_device *dev,
+ struct cfg80211_chan_def *chandef)
+ {
+ struct lbs_private *priv = wiphy_priv(wiphy);
+diff --git a/drivers/net/wireless/marvell/libertas/firmware.c b/drivers/net/wireless/marvell/libertas/firmware.c
+index f124110944b7e9..9bf7d4c207b9ed 100644
+--- a/drivers/net/wireless/marvell/libertas/firmware.c
++++ b/drivers/net/wireless/marvell/libertas/firmware.c
+@@ -78,6 +78,7 @@ static void helper_firmware_cb(const struct firmware *firmware, void *context)
+ } else {
+ /* No main firmware needed for this helper --> success! */
+ lbs_fw_loaded(priv, 0, firmware, NULL);
++ release_firmware(firmware);
+ }
+ }
+
+diff --git a/drivers/net/wireless/marvell/libertas_tf/main.c b/drivers/net/wireless/marvell/libertas_tf/main.c
+index b47a832b9ae214..25509a873c448b 100644
+--- a/drivers/net/wireless/marvell/libertas_tf/main.c
++++ b/drivers/net/wireless/marvell/libertas_tf/main.c
+@@ -174,7 +174,7 @@ static void lbtf_free_adapter(struct lbtf_private *priv)
+ {
+ lbtf_deb_enter(LBTF_DEB_MAIN);
+ lbtf_free_cmd_buffer(priv);
+- del_timer(&priv->command_timer);
++ timer_delete_sync(&priv->command_timer);
+ lbtf_deb_leave(LBTF_DEB_MAIN);
+ }
+
+diff --git a/drivers/net/wireless/marvell/mwifiex/tdls.c b/drivers/net/wireless/marvell/mwifiex/tdls.c
+index 7823e67694e888..73dc432bd82671 100644
+--- a/drivers/net/wireless/marvell/mwifiex/tdls.c
++++ b/drivers/net/wireless/marvell/mwifiex/tdls.c
+@@ -215,7 +215,7 @@ mwifiex_tdls_add_ht_oper(struct mwifiex_private *priv, const u8 *mac,
+
+ /* follow AP's channel bandwidth */
+ if (ISSUPP_CHANWIDTH40(priv->adapter->hw_dot_11n_dev_cap) &&
+- bss_desc->bcn_ht_cap &&
++ bss_desc->bcn_ht_oper &&
+ ISALLOWED_CHANWIDTH40(bss_desc->bcn_ht_oper->ht_param))
+ ht_oper->ht_param = bss_desc->bcn_ht_oper->ht_param;
+
+diff --git a/drivers/net/wireless/marvell/mwifiex/uap_event.c b/drivers/net/wireless/marvell/mwifiex/uap_event.c
+index 58ef5020a46a73..c7383abf064f50 100644
+--- a/drivers/net/wireless/marvell/mwifiex/uap_event.c
++++ b/drivers/net/wireless/marvell/mwifiex/uap_event.c
+@@ -123,11 +123,31 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
+ len = ETH_ALEN;
+
+ if (len != -1) {
++ u16 evt_len = le16_to_cpu(event->len);
++
+ sinfo->assoc_req_ies = &event->data[len];
+ len = (u8 *)sinfo->assoc_req_ies -
+ (u8 *)&event->frame_control;
+- sinfo->assoc_req_ies_len =
+- le16_to_cpu(event->len) - (u16)len;
++
++ /*
++ * event->len is reported by the device firmware
++ * and is not otherwise validated. Reject a
++ * length that underflows the header, or that
++ * would place the association request IEs
++ * outside the fixed-size event_body[] buffer the
++ * event was copied into; otherwise the IE walk
++ * in mwifiex_set_sta_ht_cap() reads past
++ * event_body and out of the adapter slab object.
++ */
++ if (evt_len < len ||
++ (u8 *)&event->frame_control + evt_len >
++ adapter->event_body + MAX_EVENT_SIZE) {
++ mwifiex_dbg(adapter, ERROR,
++ "invalid STA assoc event length\n");
++ kfree(sinfo);
++ return -1;
++ }
++ sinfo->assoc_req_ies_len = evt_len - (u16)len;
+ }
+ }
+ cfg80211_new_sta(priv->netdev, event->sta_addr, sinfo,
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
+index 1e473f490b4bd3..4f416ebabe86e9 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c
+@@ -1612,6 +1612,8 @@ bool mt7615_rx_check(struct mt76_dev *mdev, void *data, int len)
+
+ switch (type) {
+ case PKT_TYPE_TXRX_NOTIFY:
++ if (!mt76_is_mmio(mdev))
++ return false;
+ mt7615_mac_tx_free(dev, data, len);
+ return false;
+ case PKT_TYPE_TXS:
+@@ -1645,6 +1647,10 @@ void mt7615_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
+ dev_kfree_skb(skb);
+ break;
+ case PKT_TYPE_TXRX_NOTIFY:
++ if (!mt76_is_mmio(mdev)) {
++ dev_kfree_skb(skb);
++ break;
++ }
+ mt7615_mac_tx_free(dev, skb->data, skb->len);
+ dev_kfree_skb(skb);
+ break;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
+index 462b4a68c4f0f1..b1a7fbedffac5c 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
+@@ -1435,6 +1435,8 @@ mt76_connac_mcu_uni_bss_he_tlv(struct mt76_phy *phy, struct ieee80211_vif *vif,
+ struct bss_info_uni_he *he;
+
+ cap = mt76_connac_get_he_phy_cap(phy, vif);
++ if (!cap)
++ return;
+
+ he = (struct bss_info_uni_he *)tlv;
+ he->he_pe_duration = vif->bss_conf.htc_trig_based_pkt_ext;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
+index 2489364110d6eb..d622d7c7bee41c 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
+@@ -551,6 +551,8 @@ mt7915_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+ struct tlv *tlv;
+
+ cap = mt76_connac_get_he_phy_cap(phy->mt76, vif);
++ if (!cap)
++ return;
+
+ tlv = mt76_connac_mcu_add_tlv(skb, BSS_INFO_HE_BASIC, sizeof(*he));
+
+@@ -1133,13 +1135,12 @@ mt7915_mcu_sta_bfer_vht(struct ieee80211_sta *sta, struct mt7915_phy *phy,
+ }
+
+ static void
+-mt7915_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
+- struct mt7915_phy *phy, struct sta_rec_bf *bf)
++mt7915_mcu_sta_bfer_he(struct ieee80211_sta *sta,
++ const struct ieee80211_sta_he_cap *vc,
++ struct sta_rec_bf *bf)
+ {
+ struct ieee80211_sta_he_cap *pc = &sta->deflink.he_cap;
+ struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem;
+- const struct ieee80211_sta_he_cap *vc =
+- mt76_connac_get_he_phy_cap(phy->mt76, vif);
+ const struct ieee80211_he_cap_elem *ve = &vc->he_cap_elem;
+ u16 mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_80);
+ u8 nss_mcs = mt7915_mcu_get_sta_nss(mcs_map);
+@@ -1198,6 +1199,7 @@ mt7915_mcu_sta_bfer_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
+ {
+ struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv;
+ struct mt7915_phy *phy = mvif->phy;
++ const struct ieee80211_sta_he_cap *vc = NULL;
+ int tx_ant = hweight8(phy->mt76->chainmask) - 1;
+ struct sta_rec_bf *bf;
+ struct tlv *tlv;
+@@ -1216,6 +1218,12 @@ mt7915_mcu_sta_bfer_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
+ if (!ebf && !dev->ibf)
+ return;
+
++ if (sta->deflink.he_cap.has_he && ebf) {
++ vc = mt76_connac_get_he_phy_cap(phy->mt76, vif);
++ if (!vc)
++ return;
++ }
++
+ tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_BF, sizeof(*bf));
+ bf = (struct sta_rec_bf *)tlv;
+
+@@ -1224,7 +1232,7 @@ mt7915_mcu_sta_bfer_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
+ * ht: iBF only, since mac80211 lacks of eBF support
+ */
+ if (sta->deflink.he_cap.has_he && ebf)
+- mt7915_mcu_sta_bfer_he(sta, vif, phy, bf);
++ mt7915_mcu_sta_bfer_he(sta, vc, bf);
+ else if (sta->deflink.vht_cap.vht_supported)
+ mt7915_mcu_sta_bfer_vht(sta, phy, bf, ebf);
+ else if (sta->deflink.ht_cap.ht_supported)
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
+index 0c6b6d6c246ee1..6da67054e521d9 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
+@@ -580,8 +580,9 @@ bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len)
+
+ switch (type) {
+ case PKT_TYPE_TXRX_NOTIFY:
+- /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */
+- mt7921_mac_tx_free(dev, data, len); /* mmio */
++ if (!mt76_is_mmio(mdev))
++ return false;
++ mt7921_mac_tx_free(dev, data, len);
+ return false;
+ case PKT_TYPE_TXS:
+ for (rxd += 2; rxd + 8 <= end; rxd += 8)
+@@ -610,7 +611,10 @@ void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
+
+ switch (type) {
+ case PKT_TYPE_TXRX_NOTIFY:
+- /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */
++ if (!mt76_is_mmio(mdev)) {
++ napi_consume_skb(skb, 1);
++ break;
++ }
+ mt7921_mac_tx_free(dev, skb->data, skb->len);
+ napi_consume_skb(skb, 1);
+ break;
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c
+index f01dbca6e9b9ca..58b46be0aaed72 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c
+@@ -1204,8 +1204,9 @@ bool mt7925_rx_check(struct mt76_dev *mdev, void *data, int len)
+
+ switch (type) {
+ case PKT_TYPE_TXRX_NOTIFY:
+- /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */
+- mt7925_mac_tx_free(dev, data, len); /* mmio */
++ if (!mt76_is_mmio(mdev))
++ return false;
++ mt7925_mac_tx_free(dev, data, len);
+ return false;
+ case PKT_TYPE_TXS:
+ for (rxd += 4; rxd + 12 <= end; rxd += 12)
+@@ -1241,7 +1242,10 @@ void mt7925_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
+
+ switch (type) {
+ case PKT_TYPE_TXRX_NOTIFY:
+- /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */
++ if (!mt76_is_mmio(mdev)) {
++ napi_consume_skb(skb, 1);
++ break;
++ }
+ mt7925_mac_tx_free(dev, skb->data, skb->len);
+ napi_consume_skb(skb, 1);
+ break;
+@@ -1285,6 +1289,9 @@ mt7925_vif_connect_iter(void *priv, u8 *mac,
+
+ for_each_set_bit(i, &valid, IEEE80211_MLD_MAX_NUM_LINKS) {
+ bss_conf = mt792x_vif_to_bss_conf(vif, i);
++ if (!bss_conf)
++ continue;
++
+ mconf = mt792x_vif_to_link(mvif, i);
+
+ mt76_connac_mcu_uni_add_dev(&dev->mphy, bss_conf, &mconf->mt76,
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c
+index 98ec76cf2fe8b4..1dbc6d22b06661 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c
+@@ -1583,6 +1583,9 @@ static void mt7925_sta_set_decap_offload(struct ieee80211_hw *hw,
+
+ mlink = mt792x_sta_to_link(msta, i);
+
++ if (!mlink)
++ continue;
++
+ if (enabled)
+ set_bit(MT_WCID_FLAG_HDR_TRANS, &mlink->wcid.flags);
+ else
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
+index 1f113a618515e8..1fc1116d9becca 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
+@@ -2646,6 +2646,8 @@ mt7925_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_bss_conf *link_conf,
+ struct tlv *tlv;
+
+ cap = mt76_connac_get_he_phy_cap(phy->mt76, link_conf->vif);
++ if (!cap)
++ return;
+
+ tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_HE_BASIC, sizeof(*he));
+
+diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
+index 54567a5893449e..a5693efee74b85 100644
+--- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
++++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
+@@ -798,6 +798,8 @@ mt7996_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_vif *vif,
+ struct tlv *tlv;
+
+ cap = mt76_connac_get_he_phy_cap(phy->mt76, vif);
++ if (!cap)
++ return;
+
+ tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_HE_BASIC, sizeof(*he));
+
+@@ -1537,17 +1539,18 @@ mt7996_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
+ {
+ struct ieee80211_sta_he_cap *pc = &sta->deflink.he_cap;
+ struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem;
+- const struct ieee80211_sta_he_cap *vc =
+- mt76_connac_get_he_phy_cap(phy->mt76, vif);
+- const struct ieee80211_he_cap_elem *ve = &vc->he_cap_elem;
+ u16 mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_80);
+ u8 nss_mcs = mt7996_mcu_get_sta_nss(mcs_map);
++ const struct ieee80211_he_cap_elem *ve;
++ const struct ieee80211_sta_he_cap *vc;
+ u8 snd_dim, sts;
+
++ vc = mt76_connac_get_he_phy_cap(phy->mt76, vif);
+ if (!vc)
+ return;
+
+ bf->tx_mode = MT_PHY_TYPE_HE_SU;
++ ve = &vc->he_cap_elem;
+
+ mt7996_mcu_sta_sounding_rate(bf);
+
+@@ -1601,14 +1604,18 @@ mt7996_mcu_sta_bfer_eht(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
+ struct ieee80211_sta_eht_cap *pc = &sta->deflink.eht_cap;
+ struct ieee80211_eht_cap_elem_fixed *pe = &pc->eht_cap_elem;
+ struct ieee80211_eht_mcs_nss_supp *eht_nss = &pc->eht_mcs_nss_supp;
+- const struct ieee80211_sta_eht_cap *vc =
+- mt76_connac_get_eht_phy_cap(phy->mt76, vif);
+- const struct ieee80211_eht_cap_elem_fixed *ve = &vc->eht_cap_elem;
+ u8 nss_mcs = u8_get_bits(eht_nss->bw._80.rx_tx_mcs9_max_nss,
+ IEEE80211_EHT_MCS_NSS_RX) - 1;
++ const struct ieee80211_eht_cap_elem_fixed *ve;
++ const struct ieee80211_sta_eht_cap *vc;
+ u8 snd_dim, sts;
+
++ vc = mt76_connac_get_eht_phy_cap(phy->mt76, vif);
++ if (!vc)
++ return;
++
+ bf->tx_mode = MT_PHY_TYPE_EHT_MU;
++ ve = &vc->eht_cap_elem;
+
+ mt7996_mcu_sta_sounding_rate(bf);
+
+diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+index eb37b228d54ea0..9977be0020fe76 100644
+--- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c
++++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+@@ -231,6 +231,7 @@ struct wilc_vif *wilc_get_wl_to_vif(struct wilc *wl)
+ }
+
+ static int set_channel(struct wiphy *wiphy,
++ struct net_device *dev,
+ struct cfg80211_chan_def *chandef)
+ {
+ struct wilc *wl = wiphy_priv(wiphy);
+@@ -1424,7 +1425,7 @@ static int start_ap(struct wiphy *wiphy, struct net_device *dev,
+ struct wilc_vif *vif = netdev_priv(dev);
+ int ret;
+
+- ret = set_channel(wiphy, &settings->chandef);
++ ret = set_channel(wiphy, dev, &settings->chandef);
+ if (ret != 0)
+ netdev_err(dev, "Error in setting channel\n");
+
+diff --git a/drivers/net/wireless/microchip/wilc1000/hif.c b/drivers/net/wireless/microchip/wilc1000/hif.c
+index b1b0c9a133b259..4b3d630905d4e3 100644
+--- a/drivers/net/wireless/microchip/wilc1000/hif.c
++++ b/drivers/net/wireless/microchip/wilc1000/hif.c
+@@ -600,6 +600,11 @@ static s32 wilc_parse_assoc_resp_info(u8 *buffer, u32 buffer_len,
+ u16 ies_len;
+ struct wilc_assoc_resp *res = (struct wilc_assoc_resp *)buffer;
+
++ if (buffer_len < sizeof(*res)) {
++ ret_conn_info->status = WLAN_STATUS_UNSPECIFIED_FAILURE;
++ return -EINVAL;
++ }
++
+ ret_conn_info->status = le16_to_cpu(res->status_code);
+ if (ret_conn_info->status == WLAN_STATUS_SUCCESS) {
+ ies = &buffer[sizeof(*res)];
+diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c
+index e992e59b591892..63fae67c816225 100644
+--- a/drivers/net/wireless/virtual/mac80211_hwsim.c
++++ b/drivers/net/wireless/virtual/mac80211_hwsim.c
+@@ -6605,6 +6605,7 @@ static void hwsim_virtio_rx_work(struct work_struct *work)
+
+ skb->data = skb->head;
+ skb_reset_tail_pointer(skb);
++ len = min(len, skb_end_offset(skb));
+ skb_put(skb, len);
+ hwsim_virtio_handle_cmd(skb);
+
+diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
+index 72e31ef1f2f063..0f1956bb6d3c9e 100644
+--- a/drivers/nvme/target/admin-cmd.c
++++ b/drivers/nvme/target/admin-cmd.c
+@@ -1008,6 +1008,27 @@ out:
+ nvmet_req_complete(req, status);
+ }
+
++u32 nvmet_admin_cmd_data_len(struct nvmet_req *req)
++{
++ struct nvme_command *cmd = req->cmd;
++
++ if (nvme_is_fabrics(cmd))
++ return nvmet_fabrics_admin_cmd_data_len(req);
++ if (nvmet_is_disc_subsys(nvmet_req_subsys(req)))
++ return nvmet_discovery_cmd_data_len(req);
++
++ switch (cmd->common.opcode) {
++ case nvme_admin_get_log_page:
++ return nvmet_get_log_page_len(cmd);
++ case nvme_admin_identify:
++ return NVME_IDENTIFY_DATA_SIZE;
++ case nvme_admin_get_features:
++ return nvmet_feat_data_len(req, le32_to_cpu(cmd->common.cdw10));
++ default:
++ return 0;
++ }
++}
++
+ u16 nvmet_parse_admin_cmd(struct nvmet_req *req)
+ {
+ struct nvme_command *cmd = req->cmd;
+diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
+index 9dcca9d03c28a9..ce85b578326033 100644
+--- a/drivers/nvme/target/core.c
++++ b/drivers/nvme/target/core.c
+@@ -891,6 +891,33 @@ static inline u16 nvmet_io_cmd_check_access(struct nvmet_req *req)
+ return 0;
+ }
+
++static u32 nvmet_io_cmd_transfer_len(struct nvmet_req *req)
++{
++ struct nvme_command *cmd = req->cmd;
++ u32 metadata_len = 0;
++
++ if (nvme_is_fabrics(cmd))
++ return nvmet_fabrics_io_cmd_data_len(req);
++
++ if (!req->ns)
++ return 0;
++
++ switch (req->cmd->common.opcode) {
++ case nvme_cmd_read:
++ case nvme_cmd_write:
++ case nvme_cmd_zone_append:
++ if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns))
++ metadata_len = nvmet_rw_metadata_len(req);
++ return nvmet_rw_data_len(req) + metadata_len;
++ case nvme_cmd_dsm:
++ return nvmet_dsm_len(req);
++ case nvme_cmd_zone_mgmt_recv:
++ return (le32_to_cpu(req->cmd->zmr.numd) + 1) << 2;
++ default:
++ return 0;
++ }
++}
++
+ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
+ {
+ struct nvme_command *cmd = req->cmd;
+@@ -1015,6 +1042,16 @@ void nvmet_req_uninit(struct nvmet_req *req)
+ }
+ EXPORT_SYMBOL_GPL(nvmet_req_uninit);
+
++size_t nvmet_req_transfer_len(struct nvmet_req *req)
++{
++ if (likely(req->sq->qid != 0))
++ return nvmet_io_cmd_transfer_len(req);
++ if (unlikely(!req->sq->ctrl))
++ return nvmet_connect_cmd_data_len(req);
++ return nvmet_admin_cmd_data_len(req);
++}
++EXPORT_SYMBOL_GPL(nvmet_req_transfer_len);
++
+ bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len)
+ {
+ if (unlikely(len != req->transfer_len)) {
+diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c
+index 9658496cc0eaf3..61377fe6cc77dd 100644
+--- a/drivers/nvme/target/discovery.c
++++ b/drivers/nvme/target/discovery.c
+@@ -373,6 +373,20 @@ static void nvmet_execute_disc_get_features(struct nvmet_req *req)
+ nvmet_req_complete(req, stat);
+ }
+
++u32 nvmet_discovery_cmd_data_len(struct nvmet_req *req)
++{
++ struct nvme_command *cmd = req->cmd;
++
++ switch (cmd->common.opcode) {
++ case nvme_admin_get_log_page:
++ return nvmet_get_log_page_len(req->cmd);
++ case nvme_admin_identify:
++ return NVME_IDENTIFY_DATA_SIZE;
++ default:
++ return 0;
++ }
++}
++
+ u16 nvmet_parse_discovery_cmd(struct nvmet_req *req)
+ {
+ struct nvme_command *cmd = req->cmd;
+diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c
+index 9839f278a5a5f2..07f2581529d53b 100644
+--- a/drivers/nvme/target/fabrics-cmd-auth.c
++++ b/drivers/nvme/target/fabrics-cmd-auth.c
+@@ -188,6 +188,11 @@ static u8 nvmet_auth_failure2(void *d)
+ return data->rescode_exp;
+ }
+
++u32 nvmet_auth_send_data_len(struct nvmet_req *req)
++{
++ return le32_to_cpu(req->cmd->auth_send.tl);
++}
++
+ void nvmet_execute_auth_send(struct nvmet_req *req)
+ {
+ struct nvmet_ctrl *ctrl = req->sq->ctrl;
+@@ -215,7 +220,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
+ offsetof(struct nvmf_auth_send_command, spsp1);
+ goto done;
+ }
+- tl = le32_to_cpu(req->cmd->auth_send.tl);
++ tl = nvmet_auth_send_data_len(req);
+ if (!tl) {
+ status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
+ req->error_loc =
+@@ -438,6 +443,35 @@ static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al)
+ data->rescode_exp = req->sq->dhchap_status;
+ }
+
++u32 nvmet_auth_receive_data_len(struct nvmet_req *req)
++{
++ struct nvmet_ctrl *ctrl = req->sq->ctrl;
++ u32 al = le32_to_cpu(req->cmd->auth_receive.al);
++ u32 min_len;
++
++ /*
++ * Reject too-short al before kmalloc(al), since the SUCCESS1 and
++ * FAILURE1/default builders write fixed response headers into it.
++ */
++ switch (req->sq->dhchap_step) {
++ case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE:
++ return al;
++ case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1:
++ min_len = sizeof(struct nvmf_auth_dhchap_success1_data);
++ if (req->sq->dhchap_c2)
++ min_len += nvme_auth_hmac_hash_len(ctrl->shash_id);
++ break;
++ default:
++ min_len = sizeof(struct nvmf_auth_dhchap_failure_data);
++ break;
++ }
++
++ if (al < min_len)
++ return 0;
++
++ return al;
++}
++
+ void nvmet_execute_auth_receive(struct nvmet_req *req)
+ {
+ struct nvmet_ctrl *ctrl = req->sq->ctrl;
+@@ -463,7 +497,7 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
+ offsetof(struct nvmf_auth_receive_command, spsp1);
+ goto done;
+ }
+- al = le32_to_cpu(req->cmd->auth_receive.al);
++ al = nvmet_auth_receive_data_len(req);
+ if (!al) {
+ status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
+ req->error_loc =
+diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c
+index c4b2eddd5666aa..a593003c1c20c6 100644
+--- a/drivers/nvme/target/fabrics-cmd.c
++++ b/drivers/nvme/target/fabrics-cmd.c
+@@ -82,6 +82,22 @@ static void nvmet_execute_prop_get(struct nvmet_req *req)
+ nvmet_req_complete(req, status);
+ }
+
++u32 nvmet_fabrics_admin_cmd_data_len(struct nvmet_req *req)
++{
++ struct nvme_command *cmd = req->cmd;
++
++ switch (cmd->fabrics.fctype) {
++#ifdef CONFIG_NVME_TARGET_AUTH
++ case nvme_fabrics_type_auth_send:
++ return nvmet_auth_send_data_len(req);
++ case nvme_fabrics_type_auth_receive:
++ return nvmet_auth_receive_data_len(req);
++#endif
++ default:
++ return 0;
++ }
++}
++
+ u16 nvmet_parse_fabrics_admin_cmd(struct nvmet_req *req)
+ {
+ struct nvme_command *cmd = req->cmd;
+@@ -111,6 +127,22 @@ u16 nvmet_parse_fabrics_admin_cmd(struct nvmet_req *req)
+ return 0;
+ }
+
++u32 nvmet_fabrics_io_cmd_data_len(struct nvmet_req *req)
++{
++ struct nvme_command *cmd = req->cmd;
++
++ switch (cmd->fabrics.fctype) {
++#ifdef CONFIG_NVME_TARGET_AUTH
++ case nvme_fabrics_type_auth_send:
++ return nvmet_auth_send_data_len(req);
++ case nvme_fabrics_type_auth_receive:
++ return nvmet_auth_receive_data_len(req);
++#endif
++ default:
++ return 0;
++ }
++}
++
+ u16 nvmet_parse_fabrics_io_cmd(struct nvmet_req *req)
+ {
+ struct nvme_command *cmd = req->cmd;
+@@ -342,6 +374,17 @@ out_ctrl_put:
+ goto out;
+ }
+
++u32 nvmet_connect_cmd_data_len(struct nvmet_req *req)
++{
++ struct nvme_command *cmd = req->cmd;
++
++ if (!nvme_is_fabrics(cmd) ||
++ cmd->fabrics.fctype != nvme_fabrics_type_connect)
++ return 0;
++
++ return sizeof(struct nvmf_connect_data);
++}
++
+ u16 nvmet_parse_connect_cmd(struct nvmet_req *req)
+ {
+ struct nvme_command *cmd = req->cmd;
+diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
+index 31a422a3f85b88..17a63bc7d80c04 100644
+--- a/drivers/nvme/target/nvmet.h
++++ b/drivers/nvme/target/nvmet.h
+@@ -466,18 +466,24 @@ void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl);
+ void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl);
+
+ u16 nvmet_parse_connect_cmd(struct nvmet_req *req);
++u32 nvmet_connect_cmd_data_len(struct nvmet_req *req);
+ void nvmet_bdev_set_limits(struct block_device *bdev, struct nvme_id_ns *id);
+ u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req);
+ u16 nvmet_file_parse_io_cmd(struct nvmet_req *req);
+ u16 nvmet_bdev_zns_parse_io_cmd(struct nvmet_req *req);
++u32 nvmet_admin_cmd_data_len(struct nvmet_req *req);
+ u16 nvmet_parse_admin_cmd(struct nvmet_req *req);
++u32 nvmet_discovery_cmd_data_len(struct nvmet_req *req);
+ u16 nvmet_parse_discovery_cmd(struct nvmet_req *req);
+ u16 nvmet_parse_fabrics_admin_cmd(struct nvmet_req *req);
++u32 nvmet_fabrics_admin_cmd_data_len(struct nvmet_req *req);
+ u16 nvmet_parse_fabrics_io_cmd(struct nvmet_req *req);
++u32 nvmet_fabrics_io_cmd_data_len(struct nvmet_req *req);
+
+ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
+ struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops);
+ void nvmet_req_uninit(struct nvmet_req *req);
++size_t nvmet_req_transfer_len(struct nvmet_req *req);
+ bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len);
+ bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len);
+ void nvmet_req_complete(struct nvmet_req *req, u16 status);
+@@ -720,7 +726,9 @@ static inline void nvmet_req_bio_put(struct nvmet_req *req, struct bio *bio)
+ }
+
+ #ifdef CONFIG_NVME_TARGET_AUTH
++u32 nvmet_auth_send_data_len(struct nvmet_req *req);
+ void nvmet_execute_auth_send(struct nvmet_req *req);
++u32 nvmet_auth_receive_data_len(struct nvmet_req *req);
+ void nvmet_execute_auth_receive(struct nvmet_req *req);
+ int nvmet_auth_set_key(struct nvmet_host *host, const char *secret,
+ bool set_ctrl);
+diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c
+index 61b18ac206c9ee..dc8142f4fa9541 100644
+--- a/drivers/platform/loongarch/loongson-laptop.c
++++ b/drivers/platform/loongarch/loongson-laptop.c
+@@ -189,6 +189,7 @@ static int __init setup_acpi_notify(struct generic_sub_driver *sub_driver)
+
+ static int loongson_hotkey_suspend(struct device *dev)
+ {
++ bl_powered = false;
+ return 0;
+ }
+
+diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
+index e22b683a7a434c..2e4e2bcc3c6e71 100644
+--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
++++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
+@@ -240,15 +240,20 @@ int uncore_freq_add_entry(struct uncore_data *data, int cpu)
+ sprintf(data->name, "package_%02d_die_%02d", data->package_id, data->die_id);
+ }
+
++ /*
++ * Set the control CPU before any read path so entry recreation after CPU
++ * hotplug can populate read-only attributes from the new online CPU.
++ */
++ data->control_cpu = cpu;
+ uncore_read(data, &data->initial_min_freq_khz, UNCORE_INDEX_MIN_FREQ);
+ uncore_read(data, &data->initial_max_freq_khz, UNCORE_INDEX_MAX_FREQ);
+
+ ret = create_attr_group(data, data->name);
+ if (ret) {
++ data->control_cpu = -1;
+ if (data->domain_id != UNCORE_DOMAIN_ID_INVALID)
+ ida_free(&intel_uncore_ida, data->instance_id);
+ } else {
+- data->control_cpu = cpu;
+ data->valid = true;
+ }
+
+diff --git a/drivers/regulator/mt6358-regulator.c b/drivers/regulator/mt6358-regulator.c
+index e4745f616cea3b..44bef0bdfdbe35 100644
+--- a/drivers/regulator/mt6358-regulator.c
++++ b/drivers/regulator/mt6358-regulator.c
+@@ -492,7 +492,7 @@ static const struct regulator_ops mt6358_volt_fixed_ops = {
+ .list_voltage = regulator_list_voltage_linear,
+ .map_voltage = regulator_map_voltage_linear,
+ .set_voltage_sel = regulator_set_voltage_sel_regmap,
+- .get_voltage_sel = mt6358_get_buck_voltage_sel,
++ .get_voltage_sel = regulator_get_voltage_sel_regmap,
+ .set_voltage_time_sel = regulator_set_voltage_time_sel,
+ .enable = regulator_enable_regmap,
+ .disable = regulator_disable_regmap,
+diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c
+index d1c069704da8d9..88561f282f9bc4 100644
+--- a/drivers/remoteproc/xlnx_r5_remoteproc.c
++++ b/drivers/remoteproc/xlnx_r5_remoteproc.c
+@@ -927,16 +927,6 @@ static struct zynqmp_r5_core *zynqmp_r5_add_rproc_core(struct device *cdev)
+ goto free_rproc;
+ }
+
+- /*
+- * If firmware is already available in the memory then move rproc state
+- * to DETACHED. Firmware can be preloaded via debugger or by any other
+- * agent (processors) in the system.
+- * If firmware isn't available in the memory and resource table isn't
+- * found, then rproc state remains OFFLINE.
+- */
+- if (!zynqmp_r5_get_rsc_table_va(r5_core))
+- r5_rproc->state = RPROC_DETACHED;
+-
+ r5_core->rproc = r5_rproc;
+ return r5_core;
+
+@@ -1189,6 +1179,7 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster,
+ {
+ struct device *dev = cluster->dev;
+ struct zynqmp_r5_core *r5_core;
++ u32 payload[PAYLOAD_ARG_CNT];
+ int ret = -EINVAL, i;
+
+ r5_core = cluster->r5_cores[0];
+@@ -1234,6 +1225,45 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster,
+ ret = zynqmp_r5_get_sram_banks(r5_core);
+ if (ret)
+ return ret;
++
++ /*
++ * It is possible that firmware is loaded into the memory, but
++ * RPU (remote) is not running. In such case, RPU state will be
++ * moved to RPROC_DETACHED wrongfully. To avoid it first make
++ * sure RPU is power-on and out of reset before parsing for the
++ * resource table.
++ */
++ ret = zynqmp_pm_feature(PM_GET_NODE_STATUS);
++ if (ret < PM_API_VERSION_2)
++ ret = -EOPNOTSUPP;
++ else
++ ret = zynqmp_pm_invoke_fn(PM_GET_NODE_STATUS, payload, 1,
++ r5_core->pm_domain_id);
++ if (ret) {
++ dev_warn(r5_core->dev,
++ "failed to get rpu node status, err %d\n", ret);
++ continue;
++ }
++
++ /*
++ * If RPU state is power on and out of reset i.e. running, then
++ * assign RPROC_DETACHED state. If the RPU is not out of reset
++ * then do not attempt to attach to the remote processor.
++ */
++ if (payload[1] == PM_NODE_RUNNING) {
++ /*
++ * Not all the firmware that is running on the remote
++ * core is expected to have the resource table. The
++ * firmware might not use RPMsg at all, and in that case
++ * resource table becomes irrelevant. However, we still
++ * need to make sure that running core is not reported
++ * as offline. so do not decide remote core state based
++ * on the resource table availability
++ */
++ if (zynqmp_r5_get_rsc_table_va(r5_core))
++ dev_dbg(r5_core->dev, "rsc tbl not found\n");
++ r5_core->rproc->state = RPROC_DETACHED;
++ }
+ }
+
+ return 0;
+diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
+index 5cf13d019a15e9..e617b98ce84cf2 100644
+--- a/drivers/scsi/hosts.c
++++ b/drivers/scsi/hosts.c
+@@ -343,6 +343,7 @@ static void scsi_host_dev_release(struct device *dev)
+ /* Wait for functions invoked through call_rcu(&scmd->rcu, ...) */
+ rcu_barrier();
+
++ cancel_work_sync(&shost->eh_work);
+ if (shost->tmf_work_q)
+ destroy_workqueue(shost->tmf_work_q);
+ if (shost->ehandler)
+@@ -408,6 +409,7 @@ struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int priv
+ INIT_LIST_HEAD(&shost->starved_list);
+ init_waitqueue_head(&shost->host_wait);
+ mutex_init(&shost->scan_mutex);
++ INIT_WORK(&shost->eh_work, scsi_rcu_eh_wakeup);
+
+ index = ida_alloc(&host_index_ida, GFP_KERNEL);
+ if (index < 0) {
+diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
+index 9a89a94784cd0c..03fadefa1cee8e 100644
+--- a/drivers/scsi/scsi_error.c
++++ b/drivers/scsi/scsi_error.c
+@@ -73,6 +73,26 @@ void scsi_eh_wakeup(struct Scsi_Host *shost, unsigned int busy)
+ }
+ }
+
++void scsi_rcu_eh_wakeup(struct work_struct *work)
++{
++ struct Scsi_Host *shost = container_of(work, struct Scsi_Host, eh_work);
++ unsigned long flags;
++ unsigned int busy;
++
++ /*
++ * Ensure any running scsi_dec_host_busy has completed its rcu section
++ * so changes to host state and host_eh_scheduled are visible to all
++ * future calls of scsi_dec_host_busy
++ */
++ synchronize_rcu();
++
++ busy = scsi_host_busy(shost);
++
++ spin_lock_irqsave(shost->host_lock, flags);
++ scsi_eh_wakeup(shost, busy);
++ spin_unlock_irqrestore(shost->host_lock, flags);
++}
++
+ /**
+ * scsi_schedule_eh - schedule EH for SCSI host
+ * @shost: SCSI host to invoke error handling on.
+@@ -88,7 +108,7 @@ void scsi_schedule_eh(struct Scsi_Host *shost)
+ if (scsi_host_set_state(shost, SHOST_RECOVERY) == 0 ||
+ scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY) == 0) {
+ shost->host_eh_scheduled++;
+- scsi_eh_wakeup(shost, scsi_host_busy(shost));
++ queue_work(shost->tmf_work_q, &shost->eh_work);
+ }
+
+ spin_unlock_irqrestore(shost->host_lock, flags);
+diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h
+index 9fc397a9ce7a4f..7238d53883be76 100644
+--- a/drivers/scsi/scsi_priv.h
++++ b/drivers/scsi/scsi_priv.h
+@@ -93,6 +93,7 @@ extern enum blk_eh_timer_return scsi_timeout(struct request *req);
+ extern int scsi_error_handler(void *host);
+ extern enum scsi_disposition scsi_decide_disposition(struct scsi_cmnd *cmd);
+ extern void scsi_eh_wakeup(struct Scsi_Host *shost, unsigned int busy);
++extern void scsi_rcu_eh_wakeup(struct work_struct *work);
+ extern void scsi_eh_scmd_add(struct scsi_cmnd *);
+ void scsi_eh_ready_devs(struct Scsi_Host *shost,
+ struct list_head *work_q,
+diff --git a/drivers/soc/qcom/ice.c b/drivers/soc/qcom/ice.c
+index 85504a023e326a..0aa83175fbda10 100644
+--- a/drivers/soc/qcom/ice.c
++++ b/drivers/soc/qcom/ice.c
+@@ -49,6 +49,7 @@ struct qcom_ice {
+ struct device_link *link;
+
+ struct clk *core_clk;
++ struct clk *iface_clk;
+ };
+
+ static DEFINE_XARRAY(ice_handles);
+@@ -150,8 +151,13 @@ int qcom_ice_resume(struct qcom_ice *ice)
+
+ err = clk_prepare_enable(ice->core_clk);
+ if (err) {
+- dev_err(dev, "failed to enable core clock (%d)\n",
+- err);
++ dev_err(dev, "Failed to enable core clock: %d\n", err);
++ return err;
++ }
++
++ err = clk_prepare_enable(ice->iface_clk);
++ if (err) {
++ dev_err(dev, "Failed to enable iface clock: %d\n", err);
+ return err;
+ }
+
+@@ -161,6 +167,7 @@ EXPORT_SYMBOL_GPL(qcom_ice_resume);
+
+ int qcom_ice_suspend(struct qcom_ice *ice)
+ {
++ clk_disable_unprepare(ice->iface_clk);
+ clk_disable_unprepare(ice->core_clk);
+
+ return 0;
+@@ -241,11 +248,17 @@ static struct qcom_ice *qcom_ice_create(struct device *dev,
+ engine->core_clk = devm_clk_get_optional_enabled(dev, "ice_core_clk");
+ if (!engine->core_clk)
+ engine->core_clk = devm_clk_get_optional_enabled(dev, "ice");
++ if (!engine->core_clk)
++ engine->core_clk = devm_clk_get_optional_enabled(dev, "core");
+ if (!engine->core_clk)
+ engine->core_clk = devm_clk_get_enabled(dev, NULL);
+ if (IS_ERR(engine->core_clk))
+ return ERR_CAST(engine->core_clk);
+
++ engine->iface_clk = devm_clk_get_optional_enabled(dev, "iface");
++ if (IS_ERR(engine->iface_clk))
++ return ERR_CAST(engine->iface_clk);
++
+ if (!qcom_ice_check_supported(engine))
+ return ERR_PTR(-EOPNOTSUPP);
+
+diff --git a/drivers/staging/media/meson/vdec/vdec.c b/drivers/staging/media/meson/vdec/vdec.c
+index 5e5b296f93bab4..e92d70e91fb7f0 100644
+--- a/drivers/staging/media/meson/vdec/vdec.c
++++ b/drivers/staging/media/meson/vdec/vdec.c
+@@ -898,7 +898,7 @@ static int vdec_open(struct file *file)
+
+ ret = vdec_init_ctrls(sess);
+ if (ret)
+- goto err_m2m_release;
++ goto err_m2m_ctx_release;
+
+ sess->pixfmt_cap = formats[0].pixfmts_cap[0];
+ sess->fmt_out = &formats[0];
+@@ -923,6 +923,8 @@ static int vdec_open(struct file *file)
+
+ return 0;
+
++err_m2m_ctx_release:
++ v4l2_m2m_ctx_release(sess->m2m_ctx);
+ err_m2m_release:
+ v4l2_m2m_release(sess->m2m_dev);
+ err_free_sess:
+diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c
+index 992b30e5ccac05..358481d542eb2c 100644
+--- a/drivers/staging/media/sunxi/cedrus/cedrus.c
++++ b/drivers/staging/media/sunxi/cedrus/cedrus.c
+@@ -392,6 +392,7 @@ static int cedrus_open(struct file *file)
+ err_m2m_release:
+ v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+ err_free:
++ v4l2_fh_exit(&ctx->fh);
+ kfree(ctx);
+ mutex_unlock(&dev->dev_mutex);
+
+@@ -509,7 +510,7 @@ static int cedrus_probe(struct platform_device *pdev)
+ ret = video_register_device(vfd, VFL_TYPE_VIDEO, 0);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to register video device\n");
+- goto err_m2m;
++ goto err_media;
+ }
+
+ v4l2_info(&dev->v4l2_dev,
+@@ -535,7 +536,8 @@ err_m2m_mc:
+ v4l2_m2m_unregister_media_controller(dev->m2m_dev);
+ err_video:
+ video_unregister_device(&dev->vfd);
+-err_m2m:
++err_media:
++ media_device_cleanup(&dev->mdev);
+ v4l2_m2m_release(dev->m2m_dev);
+ err_v4l2:
+ v4l2_device_unregister(&dev->v4l2_dev);
+diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
+index 3e2843ef6ccec8..fc54d993b11f2a 100644
+--- a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
++++ b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
+@@ -210,6 +210,9 @@ static void _cedrus_write_ref_list(struct cedrus_ctx *ctx,
+ u8 dpb_idx;
+
+ dpb_idx = ref_list[i].index;
++ if (dpb_idx >= V4L2_H264_NUM_DPB_ENTRIES)
++ continue;
++
+ dpb = &decode->dpb[dpb_idx];
+
+ if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE))
+diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c
+index 463410349d07e6..24215499cb7860 100644
+--- a/drivers/staging/media/tegra-video/vi.c
++++ b/drivers/staging/media/tegra-video/vi.c
+@@ -80,8 +80,8 @@ static int tegra_get_format_idx_by_code(struct tegra_vi *vi,
+ static u32 tegra_get_format_fourcc_by_idx(struct tegra_vi *vi,
+ unsigned int index)
+ {
+- if (index >= vi->soc->nformats)
+- return -EINVAL;
++ if (WARN_ON_ONCE(index >= vi->soc->nformats))
++ return vi->soc->video_formats[0].fourcc;
+
+ return vi->soc->video_formats[index].fourcc;
+ }
+diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
+index 261b3c91da6a0c..fe458912e0ecf8 100644
+--- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
++++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
+@@ -679,7 +679,14 @@ u8 *rtw_get_wps_ie(u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen)
+ while (cnt < in_len) {
+ eid = in_ie[cnt];
+
+- if ((eid == WLAN_EID_VENDOR_SPECIFIC) && (!memcmp(&in_ie[cnt + 2], wps_oui, 4))) {
++ if (cnt + 2 > in_len)
++ break;
++
++ if (in_ie[cnt + 1] + 2 > in_len - cnt)
++ break;
++
++ if ((eid == WLAN_EID_VENDOR_SPECIFIC) && (in_ie[cnt + 1] >= 4) &&
++ (!memcmp(&in_ie[cnt + 2], wps_oui, 4))) {
+ wpsie_ptr = &in_ie[cnt];
+
+ if (wps_ie)
+diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+index 887816b170c37f..96d4da546725eb 100644
+--- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
++++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+@@ -1965,7 +1965,7 @@ static u8 rtw_get_chan_type(struct adapter *adapter)
+ else
+ return NL80211_CHAN_NO_HT;
+ case CHANNEL_WIDTH_40:
+- if (mlme_ext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER)
++ if (mlme_ext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER)
+ return NL80211_CHAN_HT40PLUS;
+ else
+ return NL80211_CHAN_HT40MINUS;
+diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c
+index f88809ff370b73..82656645b8a64b 100644
+--- a/drivers/tty/serial/8250/8250_mid.c
++++ b/drivers/tty/serial/8250/8250_mid.c
+@@ -318,9 +318,11 @@ static int mid8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ if (!uart.port.membase)
+ return -ENOMEM;
+
+- ret = mid->board->setup(mid, &uart.port);
+- if (ret)
+- return ret;
++ if (mid->board->setup) {
++ ret = mid->board->setup(mid, &uart.port);
++ if (ret)
++ return ret;
++ }
+
+ ret = mid8250_dma_setup(mid, &uart);
+ if (ret)
+@@ -336,7 +338,8 @@ static int mid8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+ return 0;
+
+ err:
+- mid->board->exit(mid);
++ if (mid->board->exit)
++ mid->board->exit(mid);
+ return ret;
+ }
+
+@@ -346,7 +349,8 @@ static void mid8250_remove(struct pci_dev *pdev)
+
+ serial8250_unregister_port(mid->line);
+
+- mid->board->exit(mid);
++ if (mid->board->exit)
++ mid->board->exit(mid);
+ }
+
+ static const struct mid8250_board pnw_board = {
+diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
+index 8ed8ee58e31809..01de811bc15015 100644
+--- a/drivers/tty/serial/sc16is7xx.c
++++ b/drivers/tty/serial/sc16is7xx.c
+@@ -1302,6 +1302,17 @@ static void sc16is7xx_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
+ val ? BIT(offset) : 0);
+ }
+
++static int sc16is7xx_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
++{
++ struct sc16is7xx_port *s = gpiochip_get_data(chip);
++ struct uart_port *port = &s->p[0].port;
++ unsigned int val;
++
++ val = sc16is7xx_port_read(port, SC16IS7XX_IODIR_REG);
++
++ return val & BIT(offset) ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
++}
++
+ static int sc16is7xx_gpio_direction_input(struct gpio_chip *chip,
+ unsigned offset)
+ {
+@@ -1379,6 +1390,7 @@ static int sc16is7xx_setup_gpio_chip(struct sc16is7xx_port *s)
+ s->gpio.parent = dev;
+ s->gpio.label = dev_name(dev);
+ s->gpio.init_valid_mask = sc16is7xx_gpio_init_valid_mask;
++ s->gpio.get_direction = sc16is7xx_gpio_get_direction;
+ s->gpio.direction_input = sc16is7xx_gpio_direction_input;
+ s->gpio.get = sc16is7xx_gpio_get;
+ s->gpio.direction_output = sc16is7xx_gpio_direction_output;
+diff --git a/drivers/uio/uio_hv_generic.c b/drivers/uio/uio_hv_generic.c
+index aa7593cea2e30e..56ccbcaadc18cb 100644
+--- a/drivers/uio/uio_hv_generic.c
++++ b/drivers/uio/uio_hv_generic.c
+@@ -395,9 +395,15 @@ hv_uio_remove(struct hv_device *dev)
+ vmbus_free_ring(dev->channel);
+ }
+
++static const struct hv_vmbus_device_id hv_uio_id_table[] = {
++ { HV_FCOPY_GUID },
++ {}
++};
++MODULE_DEVICE_TABLE(vmbus, hv_uio_id_table);
++
+ static struct hv_driver hv_uio_drv = {
+ .name = "uio_hv_generic",
+- .id_table = NULL, /* only dynamic id's */
++ .id_table = hv_uio_id_table,
+ .probe = hv_uio_probe,
+ .remove = hv_uio_remove,
+ };
+diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c
+index cd53d36aa30748..dad42820814022 100644
+--- a/drivers/usb/atm/ueagle-atm.c
++++ b/drivers/usb/atm/ueagle-atm.c
+@@ -2590,6 +2590,7 @@ static struct usbatm_driver uea_usbatm_driver = {
+ static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id)
+ {
+ struct usb_device *usb = interface_to_usbdev(intf);
++ bool single_iface = usb->config->desc.bNumInterfaces == 1;
+ int ret;
+
+ uea_enters(usb);
+@@ -2599,6 +2600,22 @@ static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id)
+ le16_to_cpu(usb->descriptor.bcdDevice),
+ chip_name[UEA_CHIP_VERSION(id)]);
+
++ /*
++ * uea_probe() decides between the pre-firmware and post-firmware case
++ * from the USB id and stores a different object as interface data in
++ * each case: a struct completion for a pre-firmware device, a struct
++ * usbatm_data for a post-firmware one. uea_disconnect() instead tells
++ * the two apart by the number of interfaces (a pre-firmware device
++ * exposes a single interface, ADI930 has 2 and eagle has 3). A crafted
++ * device advertising a pre-firmware id together with a multi-interface
++ * descriptor (or the other way around) makes the two disagree, so that
++ * usbatm_usb_disconnect() treats the small completion object as a
++ * struct usbatm_data and reads out of bounds. Reject such inconsistent
++ * descriptors so both paths make the same decision.
++ */
++ if (UEA_IS_PREFIRM(id) != single_iface)
++ return -ENODEV;
++
+ usb_reset_device(usb);
+
+ if (UEA_IS_PREFIRM(id)) {
+diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c
+index ac7c58b69e9d8b..c0eaebae7ef1f2 100644
+--- a/drivers/usb/chipidea/core.c
++++ b/drivers/usb/chipidea/core.c
+@@ -1265,6 +1265,7 @@ static void ci_hdrc_remove(struct platform_device *pdev)
+ usb_role_switch_unregister(ci->role_switch);
+
+ if (ci->supports_runtime_pm) {
++ pm_runtime_dont_use_autosuspend(&pdev->dev);
+ pm_runtime_get_sync(&pdev->dev);
+ pm_runtime_disable(&pdev->dev);
+ pm_runtime_put_noidle(&pdev->dev);
+diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c
+index a3c15861a397fd..786c94885d445c 100644
+--- a/drivers/usb/core/port.c
++++ b/drivers/usb/core/port.c
+@@ -724,6 +724,8 @@ static void connector_unbind(struct device *dev, struct device *connector, void
+
+ sysfs_remove_link(&connector->kobj, dev_name(dev));
+ sysfs_remove_link(&dev->kobj, "connector");
++ if (port_dev->child)
++ typec_deattach(port_dev->connector, &port_dev->child->dev);
+ port_dev->connector = NULL;
+ }
+
+diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c
+index 61b6d978892c79..2e341cbbbaf5fa 100644
+--- a/drivers/usb/core/sysfs.c
++++ b/drivers/usb/core/sysfs.c
+@@ -899,10 +899,15 @@ bos_descriptors_read(struct file *filp, struct kobject *kobj,
+ {
+ struct device *dev = kobj_to_dev(kobj);
+ struct usb_device *udev = to_usb_device(dev);
+- struct usb_host_bos *bos = udev->bos;
++ struct usb_host_bos *bos;
+ struct usb_bos_descriptor *desc;
+ size_t desclen, n = 0;
++ int rc;
+
++ rc = usb_lock_device_interruptible(udev);
++ if (rc < 0)
++ return -EINTR;
++ bos = udev->bos;
+ if (bos) {
+ desc = bos->desc;
+ desclen = le16_to_cpu(desc->wTotalLength);
+@@ -911,6 +916,7 @@ bos_descriptors_read(struct file *filp, struct kobject *kobj,
+ memcpy(buf, (void *) desc + off, n);
+ }
+ }
++ usb_unlock_device(udev);
+ return n;
+ }
+ static BIN_ATTR_RO(bos_descriptors, 65535); /* max-size BOS */
+diff --git a/drivers/usb/gadget/function/f_ecm.c b/drivers/usb/gadget/function/f_ecm.c
+index 6a9f0afc0bb79e..e315f18b7f9f04 100644
+--- a/drivers/usb/gadget/function/f_ecm.c
++++ b/drivers/usb/gadget/function/f_ecm.c
+@@ -14,6 +14,7 @@
+ #include <linux/module.h>
+ #include <linux/device.h>
+ #include <linux/etherdevice.h>
++#include <linux/string_choices.h>
+
+ #include <linux/usb/gadget.h>
+
+@@ -390,8 +391,7 @@ static void ecm_do_notify(struct f_ecm *ecm)
+ event->wLength = 0;
+ req->length = sizeof *event;
+
+- DBG(cdev, "notify connect %s\n",
+- ecm->is_open ? "true" : "false");
++ DBG(cdev, "notify connect %s\n", str_true_false(ecm->is_open));
+ ecm->notify_state = ECM_NOTIFY_SPEED;
+ break;
+
+diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c
+index 1f18f15dba2778..bbd90d0c2b6b87 100644
+--- a/drivers/usb/gadget/function/f_midi.c
++++ b/drivers/usb/gadget/function/f_midi.c
+@@ -1302,6 +1302,7 @@ static void f_midi_free(struct usb_function *f)
+ opts = container_of(f->fi, struct f_midi_opts, func_inst);
+ mutex_lock(&opts->lock);
+ if (!--midi->free_ref) {
++ cancel_work_sync(&midi->work);
+ kfree(midi->id);
+ kfifo_free(&midi->in_req_fifo);
+ kfree(midi);
+diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c
+index 12e18221f320a6..a6a14500b65051 100644
+--- a/drivers/usb/gadget/function/f_ncm.c
++++ b/drivers/usb/gadget/function/f_ncm.c
+@@ -18,6 +18,7 @@
+ #include <linux/device.h>
+ #include <linux/etherdevice.h>
+ #include <linux/crc32.h>
++#include <linux/string_choices.h>
+
+ #include <linux/usb/cdc.h>
+ #include <linux/usb/gadget.h>
+@@ -560,7 +561,7 @@ static void ncm_do_notify(struct f_ncm *ncm)
+ req->length = sizeof *event;
+
+ DBG(cdev, "notify connect %s\n",
+- ncm->is_open ? "true" : "false");
++ str_true_false(ncm->is_open));
+ ncm->notify_state = NCM_NOTIFY_NONE;
+ break;
+
+@@ -1188,6 +1189,10 @@ static int ncm_unwrap_ntb(struct gether *port,
+ frame_max = ncm_opts->max_segment_size;
+
+ parse_ntb:
++ if (to_process < (int)opts->nth_size) {
++ INFO(port->func.config->cdev, "Packet too small for headers\n");
++ goto err;
++ }
+ tmp = (__le16 *)ntb_ptr;
+
+ /* dwSignature */
+@@ -1208,8 +1213,12 @@ parse_ntb:
+ tmp++; /* skip wSequence */
+
+ block_len = get_ncm(&tmp, opts->block_length);
++ if (block_len == 0)
++ block_len = to_process;
++
+ /* (d)wBlockLength */
+- if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max)) {
++ if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max) ||
++ (block_len > to_process)) {
+ INFO(port->func.config->cdev, "Bad block length: %#X\n", block_len);
+ goto err;
+ }
+@@ -1272,7 +1281,7 @@ parse_ntb:
+ index = index2;
+ /* wDatagramIndex[0] */
+ if ((index < opts->nth_size) ||
+- (index > block_len - opts->dpe_size)) {
++ (index > block_len)) {
+ INFO(port->func.config->cdev,
+ "Bad index: %#X\n", index);
+ goto err;
+@@ -1284,7 +1293,8 @@ parse_ntb:
+ * ethernet hdr + crc or larger than max frame size
+ */
+ if ((dg_len < 14 + crc_len) ||
+- (dg_len > frame_max)) {
++ (dg_len > frame_max) ||
++ (dg_len > block_len - index)) {
+ INFO(port->func.config->cdev,
+ "Bad dgram length: %#X\n", dg_len);
+ goto err;
+@@ -1309,7 +1319,7 @@ parse_ntb:
+ dg_len2 = get_ncm(&tmp, opts->dgram_item_len);
+
+ /* wDatagramIndex[1] */
+- if (index2 > block_len - opts->dpe_size) {
++ if (index2 > block_len) {
+ INFO(port->func.config->cdev,
+ "Bad index: %#X\n", index2);
+ goto err;
+diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c
+index a3884afa42b4bc..3db07d0fba89c8 100644
+--- a/drivers/usb/gadget/function/f_printer.c
++++ b/drivers/usb/gadget/function/f_printer.c
+@@ -431,7 +431,7 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr)
+ {
+ struct printer_dev *dev = fd->private_data;
+ unsigned long flags;
+- size_t size;
++ size_t size, not_copied, copied;
+ size_t bytes_copied;
+ struct usb_request *req;
+ /* This is a pointer to the current USB rx request. */
+@@ -524,10 +524,12 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr)
+ else
+ size = len;
+
+- size -= copy_to_user(buf, current_rx_buf, size);
+- bytes_copied += size;
+- len -= size;
+- buf += size;
++ not_copied = copy_to_user(buf, current_rx_buf, size);
++ copied = size - not_copied;
++
++ bytes_copied += copied;
++ len -= copied;
++ buf += copied;
+
+ spin_lock_irqsave(&dev->lock, flags);
+
+@@ -542,6 +544,17 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr)
+ if (dev->interface < 0)
+ goto out_disabled;
+
++ if (!copied) {
++ dev->current_rx_req = current_rx_req;
++ dev->current_rx_bytes = current_rx_bytes;
++ dev->current_rx_buf = current_rx_buf;
++ spin_unlock_irqrestore(&dev->lock, flags);
++ mutex_unlock(&dev->lock_printer_io);
++ return bytes_copied ? bytes_copied : -EFAULT;
++ }
++
++ size = copied;
++
+ /* If we not returning all the data left in this RX request
+ * buffer then adjust the amount of data left in the buffer.
+ * Othewise if we are done with this RX request buffer then
+diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c
+index 2412f81f441201..4b6484cfa0bd1f 100644
+--- a/drivers/usb/gadget/function/u_serial.c
++++ b/drivers/usb/gadget/function/u_serial.c
+@@ -21,6 +21,7 @@
+ #include <linux/tty.h>
+ #include <linux/tty_flip.h>
+ #include <linux/slab.h>
++#include <linux/string_choices.h>
+ #include <linux/export.h>
+ #include <linux/module.h>
+ #include <linux/console.h>
+@@ -1541,7 +1542,7 @@ static int __init userial_init(void)
+
+ pr_debug("%s: registered %d ttyGS* device%s\n", __func__,
+ MAX_U_SERIAL_PORTS,
+- (MAX_U_SERIAL_PORTS == 1) ? "" : "s");
++ str_plural(MAX_U_SERIAL_PORTS));
+
+ return status;
+ fail:
+diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c
+index 77e17d23ffe5f5..624215508ed682 100644
+--- a/drivers/usb/gadget/function/uvc_v4l2.c
++++ b/drivers/usb/gadget/function/uvc_v4l2.c
+@@ -191,6 +191,8 @@ uvc_send_response(struct uvc_device *uvc, struct uvc_request_data *data)
+ return usb_ep_set_halt(cdev->gadget->ep0);
+
+ req->length = min_t(unsigned int, uvc->event_length, data->length);
++ if (req->length > sizeof(data->data))
++ req->length = sizeof(data->data);
+ req->zero = data->length < uvc->event_length;
+
+ memcpy(req->buf, data->data, req->length);
+diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
+index 9c7381661016da..b6a30d88a80034 100644
+--- a/drivers/usb/gadget/legacy/inode.c
++++ b/drivers/usb/gadget/legacy/inode.c
+@@ -20,6 +20,7 @@
+ #include <linux/uaccess.h>
+ #include <linux/sched.h>
+ #include <linux/slab.h>
++#include <linux/string_choices.h>
+ #include <linux/poll.h>
+ #include <linux/kthread.h>
+ #include <linux/aio.h>
+@@ -1182,7 +1183,7 @@ ep0_fasync (int f, struct file *fd, int on)
+ {
+ struct dev_data *dev = fd->private_data;
+ // caller must F_SETOWN before signal delivery happens
+- VDEBUG (dev, "%s %s\n", __func__, on ? "on" : "off");
++ VDEBUG(dev, "%s %s\n", __func__, str_on_off(on));
+ return fasync_helper (f, fd, on, &dev->fasync);
+ }
+
+diff --git a/drivers/usb/gadget/udc/aspeed-vhub/hub.c b/drivers/usb/gadget/udc/aspeed-vhub/hub.c
+index a63e4af60a56a9..02fe1a08d5757e 100644
+--- a/drivers/usb/gadget/udc/aspeed-vhub/hub.c
++++ b/drivers/usb/gadget/udc/aspeed-vhub/hub.c
+@@ -22,6 +22,7 @@
+ #include <linux/usb/gadget.h>
+ #include <linux/of.h>
+ #include <linux/regmap.h>
++#include <linux/string_choices.h>
+ #include <linux/dma-mapping.h>
+ #include <linux/bcd.h>
+ #include <linux/version.h>
+@@ -219,7 +220,7 @@ static int ast_vhub_hub_dev_feature(struct ast_vhub_ep *ep,
+ if (wValue == USB_DEVICE_REMOTE_WAKEUP) {
+ ep->vhub->wakeup_en = is_set;
+ EPDBG(ep, "Hub remote wakeup %s\n",
+- is_set ? "enabled" : "disabled");
++ str_enabled_disabled(is_set));
+ return std_req_complete;
+ }
+
+diff --git a/drivers/usb/gadget/udc/at91_udc.c b/drivers/usb/gadget/udc/at91_udc.c
+index e3bf17a98b3806..fd859fd1c57fd2 100644
+--- a/drivers/usb/gadget/udc/at91_udc.c
++++ b/drivers/usb/gadget/udc/at91_udc.c
+@@ -16,6 +16,7 @@
+ #include <linux/delay.h>
+ #include <linux/ioport.h>
+ #include <linux/slab.h>
++#include <linux/string_choices.h>
+ #include <linux/errno.h>
+ #include <linux/list.h>
+ #include <linux/interrupt.h>
+@@ -131,7 +132,7 @@ static void proc_ep_show(struct seq_file *s, struct at91_ep *ep)
+ seq_printf(s, "csr %08x rxbytes=%d %s %s %s" EIGHTBITS "\n",
+ csr,
+ (csr & 0x07ff0000) >> 16,
+- (csr & (1 << 15)) ? "enabled" : "disabled",
++ str_enabled_disabled(csr & (1 << 15)),
+ (csr & (1 << 11)) ? "DATA1" : "DATA0",
+ types[(csr & 0x700) >> 8],
+
+diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c
+index 7fded329076cc8..e21b4c917cfbf6 100644
+--- a/drivers/usb/gadget/udc/bdc/bdc_core.c
++++ b/drivers/usb/gadget/udc/bdc/bdc_core.c
+@@ -586,9 +586,29 @@ disable_clk:
+ static void bdc_remove(struct platform_device *pdev)
+ {
+ struct bdc *bdc;
++ unsigned long flags;
++ u32 temp;
+
+ bdc = platform_get_drvdata(pdev);
+ dev_dbg(bdc->dev, "%s ()\n", __func__);
++ /*
++ * Disable the device interrupt source before freeing the IRQ:
++ * clear BDC_GIE so the controller stops asserting interrupts,
++ * then free_irq drains any in-flight handler.
++ */
++ spin_lock_irqsave(&bdc->lock, flags);
++ temp = bdc_readl(bdc->regs, BDC_BDCSC);
++ temp &= ~BDC_GIE;
++ bdc_writel(bdc->regs, BDC_BDCSC, temp);
++ spin_unlock_irqrestore(&bdc->lock, flags);
++ free_irq(bdc->irq, bdc);
++ /*
++ * Drain func_wake_notify after free_irq: the IRQ handler arms this
++ * delayed_work via bdc_sr_uspc -> handle_link_state_change ->
++ * schedule_delayed_work (self-rearmed in bdc_func_wake_timer), so
++ * the IRQ must be released first to prevent re-arm after cancel.
++ */
++ cancel_delayed_work_sync(&bdc->func_wake_notify);
+ bdc_udc_exit(bdc);
+ bdc_hw_exit(bdc);
+ bdc_phy_exit(bdc);
+diff --git a/drivers/usb/gadget/udc/bdc/bdc_udc.c b/drivers/usb/gadget/udc/bdc/bdc_udc.c
+index 23826fd7a8e693..7a12219edac6fd 100644
+--- a/drivers/usb/gadget/udc/bdc/bdc_udc.c
++++ b/drivers/usb/gadget/udc/bdc/bdc_udc.c
+@@ -530,8 +530,8 @@ int bdc_udc_init(struct bdc *bdc)
+
+
+ bdc->gadget.name = BRCM_BDC_NAME;
+- ret = devm_request_irq(bdc->dev, bdc->irq, bdc_udc_interrupt,
+- IRQF_SHARED, BRCM_BDC_NAME, bdc);
++ ret = request_irq(bdc->irq, bdc_udc_interrupt, IRQF_SHARED,
++ BRCM_BDC_NAME, bdc);
+ if (ret) {
+ dev_err(bdc->dev,
+ "failed to request irq #%d %d\n",
+@@ -542,7 +542,7 @@ int bdc_udc_init(struct bdc *bdc)
+ ret = bdc_init_ep(bdc);
+ if (ret) {
+ dev_err(bdc->dev, "bdc init ep fail: %d\n", ret);
+- return ret;
++ goto err0;
+ }
+
+ ret = usb_add_gadget_udc(bdc->dev, &bdc->gadget);
+@@ -571,6 +571,7 @@ int bdc_udc_init(struct bdc *bdc)
+ err1:
+ usb_del_gadget_udc(&bdc->gadget);
+ err0:
++ free_irq(bdc->irq, bdc);
+ bdc_free_ep(bdc);
+
+ return ret;
+diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
+index 62fce42ef2dabd..7e69944ef18a91 100644
+--- a/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
++++ b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
+@@ -29,6 +29,7 @@
+ #include <linux/pm_runtime.h>
+ #include <linux/interrupt.h>
+ #include <linux/property.h>
++#include <linux/string_choices.h>
+ #include <linux/dmapool.h>
+ #include <linux/iopoll.h>
+
+@@ -2233,12 +2234,12 @@ static int cdns2_init_eps(struct cdns2_device *pdev)
+ dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, "
+ "BULK: %s, ISOC %s, SupDir IN: %s, OUT: %s\n",
+ pep->name,
+- (pep->endpoint.caps.type_control) ? "yes" : "no",
+- (pep->endpoint.caps.type_int) ? "yes" : "no",
+- (pep->endpoint.caps.type_bulk) ? "yes" : "no",
+- (pep->endpoint.caps.type_iso) ? "yes" : "no",
+- (pep->endpoint.caps.dir_in) ? "yes" : "no",
+- (pep->endpoint.caps.dir_out) ? "yes" : "no");
++ str_yes_no(pep->endpoint.caps.type_control),
++ str_yes_no(pep->endpoint.caps.type_int),
++ str_yes_no(pep->endpoint.caps.type_bulk),
++ str_yes_no(pep->endpoint.caps.type_iso),
++ str_yes_no(pep->endpoint.caps.dir_in),
++ str_yes_no(pep->endpoint.caps.dir_out));
+
+ INIT_LIST_HEAD(&pep->pending_list);
+ INIT_LIST_HEAD(&pep->deferred_list);
+diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
+index 1c204ef5f5446a..9c8ab106662585 100644
+--- a/drivers/usb/gadget/udc/dummy_hcd.c
++++ b/drivers/usb/gadget/udc/dummy_hcd.c
+@@ -28,6 +28,7 @@
+ #include <linux/delay.h>
+ #include <linux/ioport.h>
+ #include <linux/slab.h>
++#include <linux/string_choices.h>
+ #include <linux/errno.h>
+ #include <linux/init.h>
+ #include <linux/hrtimer.h>
+@@ -277,6 +278,7 @@ struct dummy {
+ unsigned ints_enabled:1;
+ unsigned udc_suspended:1;
+ unsigned pullup:1;
++ unsigned fifo_req_busy:1;
+
+ /*
+ * HOST side support
+@@ -328,6 +330,26 @@ static inline struct dummy *gadget_dev_to_dummy(struct device *dev)
+
+ /* DEVICE/GADGET SIDE UTILITY ROUTINES */
+
++/*
++ * Give back a gadget request with dum->lock dropped around the callback.
++ * If @req is the shared fifo_req, clear fifo_req_busy afterward: the flag
++ * was set in dummy_queue() when the shared request was taken and must stay
++ * set until its completion callback has returned; list_del_init() alone
++ * makes the request look idle while the callback is still running.
++ * Caller holds dum->lock and has already done list_del_init() + status.
++ */
++static void dummy_giveback(struct dummy *dum, struct usb_ep *_ep,
++ struct dummy_request *req)
++{
++ bool fifo = req == &dum->fifo_req;
++
++ spin_unlock(&dum->lock);
++ usb_gadget_giveback_request(_ep, &req->req);
++ spin_lock(&dum->lock);
++ if (fifo)
++ dum->fifo_req_busy = 0;
++}
++
+ /* called with spinlock held */
+ static void nuke(struct dummy *dum, struct dummy_ep *ep)
+ {
+@@ -338,9 +360,7 @@ static void nuke(struct dummy *dum, struct dummy_ep *ep)
+ list_del_init(&req->queue);
+ req->req.status = -ESHUTDOWN;
+
+- spin_unlock(&dum->lock);
+- usb_gadget_giveback_request(&ep->ep, &req->req);
+- spin_lock(&dum->lock);
++ dummy_giveback(dum, &ep->ep, req);
+ }
+ }
+
+@@ -630,7 +650,7 @@ static int dummy_enable(struct usb_ep *_ep,
+ desc->bEndpointAddress & 0x0f,
+ (desc->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
+ usb_ep_type_string(usb_endpoint_type(desc)),
+- max, ep->stream_en ? "enabled" : "disabled");
++ max, str_enabled_disabled(ep->stream_en));
+
+ /* at this point real hardware should be NAKing transfers
+ * to that endpoint, until a buffer is queued to it.
+@@ -727,10 +747,11 @@ static int dummy_queue(struct usb_ep *_ep, struct usb_request *_req,
+
+ /* implement an emulated single-request FIFO */
+ if (ep->desc && (ep->desc->bEndpointAddress & USB_DIR_IN) &&
+- list_empty(&dum->fifo_req.queue) &&
++ !dum->fifo_req_busy &&
+ list_empty(&ep->queue) &&
+ _req->length <= FIFO_SIZE) {
+ req = &dum->fifo_req;
++ dum->fifo_req_busy = 1;
+ req->req = *_req;
+ req->req.buf = dum->fifo_buf;
+ memcpy(dum->fifo_buf, _req->buf, _req->length);
+@@ -784,9 +805,7 @@ static int dummy_dequeue(struct usb_ep *_ep, struct usb_request *_req)
+ dev_dbg(udc_dev(dum),
+ "dequeued req %p from %s, len %d buf %p\n",
+ req, _ep->name, _req->length, _req->buf);
+- spin_unlock(&dum->lock);
+- usb_gadget_giveback_request(_ep, _req);
+- spin_lock(&dum->lock);
++ dummy_giveback(dum, _ep, req);
+ }
+ spin_unlock_irqrestore(&dum->lock, flags);
+ return retval;
+@@ -1522,9 +1541,7 @@ top:
+ if (req->req.status != -EINPROGRESS) {
+ list_del_init(&req->queue);
+
+- spin_unlock(&dum->lock);
+- usb_gadget_giveback_request(&ep->ep, &req->req);
+- spin_lock(&dum->lock);
++ dummy_giveback(dum, &ep->ep, req);
+
+ /* requests might have been unlinked... */
+ rescan = 1;
+@@ -1908,9 +1925,7 @@ restart:
+ dev_dbg(udc_dev(dum), "stale req = %p\n",
+ req);
+
+- spin_unlock(&dum->lock);
+- usb_gadget_giveback_request(&ep->ep, &req->req);
+- spin_lock(&dum->lock);
++ dummy_giveback(dum, &ep->ep, req);
+ ep->already_seen = 0;
+ goto restart;
+ }
+diff --git a/drivers/usb/gadget/udc/fsl_udc_core.c b/drivers/usb/gadget/udc/fsl_udc_core.c
+index 0cabd4eee6acb5..a3cce8ca666055 100644
+--- a/drivers/usb/gadget/udc/fsl_udc_core.c
++++ b/drivers/usb/gadget/udc/fsl_udc_core.c
+@@ -22,6 +22,7 @@
+ #include <linux/errno.h>
+ #include <linux/err.h>
+ #include <linux/slab.h>
++#include <linux/string_choices.h>
+ #include <linux/init.h>
+ #include <linux/list.h>
+ #include <linux/interrupt.h>
+@@ -183,7 +184,7 @@ __acquires(ep->udc->lock)
+ usb_gadget_unmap_request(&ep->udc->gadget, &req->req, ep_is_in(ep));
+
+ if (status && (status != -ESHUTDOWN))
+- dev_vdbg(&udc->gadget.dev, "complete %s req %p stat %d len %u/%u\n",
++ dev_vdbg(udc->dev, "complete %s req %p stat %d len %u/%u\n",
+ ep->ep.name, &req->req, status,
+ req->req.actual, req->req.length);
+
+@@ -285,7 +286,7 @@ static int dr_controller_setup(struct fsl_udc *udc)
+ timeout = jiffies + FSL_UDC_RESET_TIMEOUT;
+ while (fsl_readl(&dr_regs->usbcmd) & USB_CMD_CTRL_RESET) {
+ if (time_after(jiffies, timeout)) {
+- dev_err(&udc->gadget.dev, "udc reset timeout!\n");
++ dev_err(udc->dev, "udc reset timeout!\n");
+ return -ETIMEDOUT;
+ }
+ cpu_relax();
+@@ -308,7 +309,7 @@ static int dr_controller_setup(struct fsl_udc *udc)
+ tmp &= USB_EP_LIST_ADDRESS_MASK;
+ fsl_writel(tmp, &dr_regs->endpointlistaddr);
+
+- dev_vdbg(&udc->gadget.dev,
++ dev_vdbg(udc->dev,
+ "vir[qh_base] is %p phy[qh_base] is 0x%8x reg is 0x%8x\n",
+ udc->ep_qh, (int)tmp,
+ fsl_readl(&dr_regs->endpointlistaddr));
+@@ -499,7 +500,7 @@ static void struct_ep_qh_setup(struct fsl_udc *udc, unsigned char ep_num,
+ tmp = max_pkt_len << EP_QUEUE_HEAD_MAX_PKT_LEN_POS;
+ break;
+ default:
+- dev_vdbg(&udc->gadget.dev, "error ep type is %d\n", ep_type);
++ dev_vdbg(udc->dev, "error ep type is %d\n", ep_type);
+ return;
+ }
+ if (zlt)
+@@ -612,7 +613,7 @@ static int fsl_ep_enable(struct usb_ep *_ep,
+ spin_unlock_irqrestore(&udc->lock, flags);
+ retval = 0;
+
+- dev_vdbg(&udc->gadget.dev, "enabled %s (ep%d%s) maxpacket %d\n",
++ dev_vdbg(udc->dev, "enabled %s (ep%d%s) maxpacket %d\n",
+ ep->ep.name, ep->ep.desc->bEndpointAddress & 0x0f,
+ (desc->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
+ max);
+@@ -633,13 +634,8 @@ static int fsl_ep_disable(struct usb_ep *_ep)
+ int ep_num;
+
+ ep = container_of(_ep, struct fsl_ep, ep);
+- if (!_ep || !ep->ep.desc) {
+- /*
+- * dev_vdbg(&udc->gadget.dev, "%s not enabled\n",
+- * _ep ? ep->ep.name : NULL);
+- */
++ if (!_ep || !ep->ep.desc)
+ return -EINVAL;
+- }
+
+ /* disable ep on controller */
+ ep_num = ep_index(ep);
+@@ -663,7 +659,7 @@ static int fsl_ep_disable(struct usb_ep *_ep)
+ ep->stopped = 1;
+ spin_unlock_irqrestore(&udc->lock, flags);
+
+- dev_vdbg(&udc->gadget.dev, "disabled %s OK\n", _ep->name);
++ dev_vdbg(udc->dev, "disabled %s OK\n", _ep->name);
+ return 0;
+ }
+
+@@ -723,9 +719,6 @@ static void fsl_queue_td(struct fsl_ep *ep, struct fsl_req *req)
+ {
+ u32 temp, bitmask, tmp_stat;
+
+- /* dev_vdbg(&udc->gadget.dev, "QH addr Register 0x%8x\n", dr_regs->endpointlistaddr);
+- dev_vdbg(&udc->gadget.dev, "ep_qh[%d] addr is 0x%8x\n", i, (u32)&(ep->udc->ep_qh[i])); */
+-
+ bitmask = ep_is_in(ep)
+ ? (1 << (ep_index(ep) + 16))
+ : (1 << (ep_index(ep)));
+@@ -812,7 +805,7 @@ static struct ep_td_struct *fsl_build_dtd(struct fsl_req *req, unsigned *length,
+ *is_last = 0;
+
+ if ((*is_last) == 0)
+- dev_vdbg(&udc_controller->gadget.dev, "multi-dtd request!\n");
++ dev_vdbg(udc_controller->dev, "multi-dtd request!\n");
+ /* Fill in the transfer size; set active bit */
+ swap_temp = ((*length << DTD_LENGTH_BIT_POS) | DTD_STATUS_ACTIVE);
+
+@@ -824,7 +817,7 @@ static struct ep_td_struct *fsl_build_dtd(struct fsl_req *req, unsigned *length,
+
+ mb();
+
+- dev_vdbg(&udc_controller->gadget.dev, "length = %d address= 0x%x\n", *length, (int)*dma);
++ dev_vdbg(udc_controller->dev, "length = %d address= 0x%x\n", *length, (int)*dma);
+
+ return dtd;
+ }
+@@ -875,11 +868,11 @@ fsl_ep_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
+ /* catch various bogus parameters */
+ if (!_req || !req->req.complete || !req->req.buf
+ || !list_empty(&req->queue)) {
+- dev_vdbg(&udc->gadget.dev, "%s, bad params\n", __func__);
++ dev_vdbg(udc->dev, "%s, bad params\n", __func__);
+ return -EINVAL;
+ }
+ if (unlikely(!ep->ep.desc)) {
+- dev_vdbg(&udc->gadget.dev, "%s, bad ep\n", __func__);
++ dev_vdbg(udc->dev, "%s, bad ep\n", __func__);
+ return -EINVAL;
+ }
+ if (usb_endpoint_xfer_isoc(ep->ep.desc)) {
+@@ -1039,7 +1032,7 @@ static int fsl_ep_set_halt(struct usb_ep *_ep, int value)
+ udc->ep0_dir = 0;
+ }
+ out:
+- dev_vdbg(&udc->gadget.dev, "%s %s halt stat %d\n", ep->ep.name,
++ dev_vdbg(udc->dev, "%s %s halt stat %d\n", ep->ep.name,
+ value ? "set" : "clear", status);
+
+ return status;
+@@ -1108,7 +1101,7 @@ static void fsl_ep_fifo_flush(struct usb_ep *_ep)
+ /* Wait until flush complete */
+ while (fsl_readl(&dr_regs->endptflush)) {
+ if (time_after(jiffies, timeout)) {
+- dev_err(&udc_controller->gadget.dev,
++ dev_err(udc_controller->dev,
+ "ep flush timeout\n");
+ return;
+ }
+@@ -1181,7 +1174,7 @@ static int fsl_vbus_session(struct usb_gadget *gadget, int is_active)
+
+ udc = container_of(gadget, struct fsl_udc, gadget);
+ spin_lock_irqsave(&udc->lock, flags);
+- dev_vdbg(&gadget->dev, "VBUS %s\n", is_active ? "on" : "off");
++ dev_vdbg(udc->dev, "VBUS %s\n", str_on_off(is_active));
+ udc->vbus_active = (is_active != 0);
+ if (can_pullup(udc))
+ fsl_writel((fsl_readl(&dr_regs->usbcmd) | USB_CMD_RUN_STOP),
+@@ -1547,7 +1540,7 @@ static void ep0_req_complete(struct fsl_udc *udc, struct fsl_ep *ep0,
+ udc->ep0_state = WAIT_FOR_SETUP;
+ break;
+ case WAIT_FOR_SETUP:
+- dev_err(&udc->gadget.dev, "Unexpected ep0 packets\n");
++ dev_err(udc->dev, "Unexpected ep0 packets\n");
+ break;
+ default:
+ ep0stall(udc);
+@@ -1616,7 +1609,7 @@ static int process_ep_req(struct fsl_udc *udc, int pipe,
+ errors = hc32_to_cpu(curr_td->size_ioc_sts);
+ if (errors & DTD_ERROR_MASK) {
+ if (errors & DTD_STATUS_HALTED) {
+- dev_err(&udc->gadget.dev, "dTD error %08x QH=%d\n", errors, pipe);
++ dev_err(udc->dev, "dTD error %08x QH=%d\n", errors, pipe);
+ /* Clear the errors and Halt condition */
+ tmp = hc32_to_cpu(curr_qh->size_ioc_int_sts);
+ tmp &= ~errors;
+@@ -1627,26 +1620,26 @@ static int process_ep_req(struct fsl_udc *udc, int pipe,
+ break;
+ }
+ if (errors & DTD_STATUS_DATA_BUFF_ERR) {
+- dev_vdbg(&udc->gadget.dev, "Transfer overflow\n");
++ dev_vdbg(udc->dev, "Transfer overflow\n");
+ status = -EPROTO;
+ break;
+ } else if (errors & DTD_STATUS_TRANSACTION_ERR) {
+- dev_vdbg(&udc->gadget.dev, "ISO error\n");
++ dev_vdbg(udc->dev, "ISO error\n");
+ status = -EILSEQ;
+ break;
+ } else
+- dev_err(&udc->gadget.dev,
++ dev_err(udc->dev,
+ "Unknown error has occurred (0x%x)!\n",
+ errors);
+
+ } else if (hc32_to_cpu(curr_td->size_ioc_sts)
+ & DTD_STATUS_ACTIVE) {
+- dev_vdbg(&udc->gadget.dev, "Request not complete\n");
++ dev_vdbg(udc->dev, "Request not complete\n");
+ status = REQ_UNCOMPLETE;
+ return status;
+ } else if (remaining_length) {
+ if (direction) {
+- dev_vdbg(&udc->gadget.dev,
++ dev_vdbg(udc->dev,
+ "Transmit dTD remaining length not zero\n");
+ status = -EPROTO;
+ break;
+@@ -1654,8 +1647,7 @@ static int process_ep_req(struct fsl_udc *udc, int pipe,
+ break;
+ }
+ } else {
+- dev_vdbg(&udc->gadget.dev,
+- "dTD transmitted successful\n");
++ dev_vdbg(udc->dev, "dTD transmitted successful\n");
+ }
+
+ if (j != curr_req->dtd_count - 1)
+@@ -1698,7 +1690,7 @@ static void dtd_complete_irq(struct fsl_udc *udc)
+
+ /* If the ep is configured */
+ if (!curr_ep->ep.name) {
+- dev_warn(&udc->gadget.dev, "Invalid EP?\n");
++ dev_warn(udc->dev, "Invalid EP?\n");
+ continue;
+ }
+
+@@ -1707,7 +1699,7 @@ static void dtd_complete_irq(struct fsl_udc *udc)
+ queue) {
+ status = process_ep_req(udc, i, curr_req);
+
+- dev_vdbg(&udc->gadget.dev,
++ dev_vdbg(udc->dev,
+ "status of process_ep_req= %d, ep = %d\n",
+ status, ep_num);
+ if (status == REQ_UNCOMPLETE)
+@@ -1828,7 +1820,7 @@ static void reset_irq(struct fsl_udc *udc)
+ while (fsl_readl(&dr_regs->endpointprime)) {
+ /* Wait until all endptprime bits cleared */
+ if (time_after(jiffies, timeout)) {
+- dev_err(&udc->gadget.dev, "Timeout for reset\n");
++ dev_err(udc->dev, "Timeout for reset\n");
+ break;
+ }
+ cpu_relax();
+@@ -1838,7 +1830,7 @@ static void reset_irq(struct fsl_udc *udc)
+ fsl_writel(0xffffffff, &dr_regs->endptflush);
+
+ if (fsl_readl(&dr_regs->portsc1) & PORTSCX_PORT_RESET) {
+- dev_vdbg(&udc->gadget.dev, "Bus reset\n");
++ dev_vdbg(udc->dev, "Bus reset\n");
+ /* Bus is reseting */
+ udc->bus_reset = 1;
+ /* Reset all the queues, include XD, dTD, EP queue
+@@ -1846,7 +1838,7 @@ static void reset_irq(struct fsl_udc *udc)
+ reset_queues(udc, true);
+ udc->usb_state = USB_STATE_DEFAULT;
+ } else {
+- dev_vdbg(&udc->gadget.dev, "Controller reset\n");
++ dev_vdbg(udc->dev, "Controller reset\n");
+ /* initialize usb hw reg except for regs for EP, not
+ * touch usbintr reg */
+ dr_controller_setup(udc);
+@@ -1880,7 +1872,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
+ /* Clear notification bits */
+ fsl_writel(irq_src, &dr_regs->usbsts);
+
+- /* dev_vdbg(&udc->gadget.dev, "irq_src [0x%8x]", irq_src); */
++ /* dev_vdbg(udc->dev, "irq_src [0x%8x]", irq_src); */
+
+ /* Need to resume? */
+ if (udc->usb_state == USB_STATE_SUSPENDED)
+@@ -1889,7 +1881,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
+
+ /* USB Interrupt */
+ if (irq_src & USB_STS_INT) {
+- dev_vdbg(&udc->gadget.dev, "Packet int\n");
++ dev_vdbg(udc->dev, "Packet int\n");
+ /* Setup package, we only support ep0 as control ep */
+ if (fsl_readl(&dr_regs->endptsetupstat) & EP_SETUP_STATUS_EP0) {
+ tripwire_handler(udc, 0,
+@@ -1918,7 +1910,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
+
+ /* Reset Received */
+ if (irq_src & USB_STS_RESET) {
+- dev_vdbg(&udc->gadget.dev, "reset int\n");
++ dev_vdbg(udc->dev, "reset int\n");
+ reset_irq(udc);
+ status = IRQ_HANDLED;
+ }
+@@ -1930,7 +1922,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
+ }
+
+ if (irq_src & (USB_STS_ERR | USB_STS_SYS_ERR)) {
+- dev_vdbg(&udc->gadget.dev, "Error IRQ %x\n", irq_src);
++ dev_vdbg(udc->dev, "Error IRQ %x\n", irq_src);
+ }
+
+ spin_unlock_irqrestore(&udc->lock, flags);
+@@ -1966,7 +1958,7 @@ static int fsl_udc_start(struct usb_gadget *g,
+ udc_controller->transceiver->otg,
+ &udc_controller->gadget);
+ if (retval < 0) {
+- dev_err(&udc_controller->gadget.dev, "can't bind to transceiver\n");
++ dev_err(udc_controller->dev, "can't bind to transceiver\n");
+ udc_controller->driver = NULL;
+ return retval;
+ }
+@@ -2251,7 +2243,7 @@ static int struct_udc_setup(struct fsl_udc *udc,
+
+ udc->eps = kcalloc(udc->max_ep, sizeof(struct fsl_ep), GFP_KERNEL);
+ if (!udc->eps) {
+- dev_err(&udc->gadget.dev, "kmalloc udc endpoint status failed\n");
++ dev_err(udc->dev, "kmalloc udc endpoint status failed\n");
+ goto eps_alloc_failed;
+ }
+
+@@ -2266,7 +2258,7 @@ static int struct_udc_setup(struct fsl_udc *udc,
+ udc->ep_qh = dma_alloc_coherent(&pdev->dev, size,
+ &udc->ep_qh_dma, GFP_KERNEL);
+ if (!udc->ep_qh) {
+- dev_err(&udc->gadget.dev, "malloc QHs for udc failed\n");
++ dev_err(udc->dev, "malloc QHs for udc failed\n");
+ goto ep_queue_alloc_failed;
+ }
+
+@@ -2277,14 +2269,14 @@ static int struct_udc_setup(struct fsl_udc *udc,
+ udc->status_req = container_of(fsl_alloc_request(NULL, GFP_KERNEL),
+ struct fsl_req, req);
+ if (!udc->status_req) {
+- dev_err(&udc->gadget.dev, "kzalloc for udc status request failed\n");
++ dev_err(udc->dev, "kzalloc for udc status request failed\n");
+ goto udc_status_alloc_failed;
+ }
+
+ /* allocate a small amount of memory to get valid address */
+ udc->status_req->req.buf = kmalloc(8, GFP_KERNEL);
+ if (!udc->status_req->req.buf) {
+- dev_err(&udc->gadget.dev, "kzalloc for udc request buffer failed\n");
++ dev_err(udc->dev, "kzalloc for udc request buffer failed\n");
+ goto udc_req_buf_alloc_failed;
+ }
+
+@@ -2372,6 +2364,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
+ if (udc_controller == NULL)
+ return -ENOMEM;
+
++ udc_controller->dev = &pdev->dev;
+ pdata = dev_get_platdata(&pdev->dev);
+ udc_controller->pdata = pdata;
+ spin_lock_init(&udc_controller->lock);
+@@ -2381,7 +2374,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
+ if (pdata->operating_mode == FSL_USB2_DR_OTG) {
+ udc_controller->transceiver = usb_get_phy(USB_PHY_TYPE_USB2);
+ if (IS_ERR_OR_NULL(udc_controller->transceiver)) {
+- dev_err(&udc_controller->gadget.dev, "Can't find OTG driver!\n");
++ dev_err(&pdev->dev, "Can't find OTG driver!\n");
+ ret = -ENODEV;
+ goto err_kfree;
+ }
+@@ -2397,7 +2390,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
+ if (pdata->operating_mode == FSL_USB2_DR_DEVICE) {
+ if (!request_mem_region(res->start, resource_size(res),
+ driver_name)) {
+- dev_err(&udc_controller->gadget.dev, "request mem region for %s failed\n", pdev->name);
++ dev_err(&pdev->dev, "failed to request mem region\n");
+ ret = -EBUSY;
+ goto err_kfree;
+ }
+@@ -2428,7 +2421,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
+ /* Read Device Controller Capability Parameters register */
+ dccparams = fsl_readl(&dr_regs->dccparams);
+ if (!(dccparams & DCCPARAMS_DC)) {
+- dev_err(&udc_controller->gadget.dev, "This SOC doesn't support device role\n");
++ dev_err(&pdev->dev, "This SOC doesn't support device role\n");
+ ret = -ENODEV;
+ goto err_exit;
+ }
+@@ -2446,14 +2439,14 @@ static int fsl_udc_probe(struct platform_device *pdev)
+ ret = request_irq(udc_controller->irq, fsl_udc_irq, IRQF_SHARED,
+ driver_name, udc_controller);
+ if (ret != 0) {
+- dev_err(&udc_controller->gadget.dev, "cannot request irq %d err %d\n",
++ dev_err(&pdev->dev, "cannot request irq %d err %d\n",
+ udc_controller->irq, ret);
+ goto err_exit;
+ }
+
+ /* Initialize the udc structure including QH member and other member */
+ if (struct_udc_setup(udc_controller, pdev)) {
+- dev_err(&udc_controller->gadget.dev, "Can't initialize udc data structure\n");
++ dev_err(&pdev->dev, "Can't initialize udc data structure\n");
+ ret = -ENOMEM;
+ goto err_free_irq;
+ }
+@@ -2473,7 +2466,6 @@ static int fsl_udc_probe(struct platform_device *pdev)
+ udc_controller->gadget.name = driver_name;
+
+ /* Setup gadget.dev and register with kernel */
+- dev_set_name(&udc_controller->gadget.dev, "gadget");
+ udc_controller->gadget.dev.of_node = pdev->dev.of_node;
+
+ if (!IS_ERR_OR_NULL(udc_controller->transceiver))
+diff --git a/drivers/usb/gadget/udc/fsl_usb2_udc.h b/drivers/usb/gadget/udc/fsl_usb2_udc.h
+index cc1756f3e89d13..53922bc58ca014 100644
+--- a/drivers/usb/gadget/udc/fsl_usb2_udc.h
++++ b/drivers/usb/gadget/udc/fsl_usb2_udc.h
+@@ -470,6 +470,7 @@ struct fsl_ep {
+ #define EP_DIR_OUT 0
+
+ struct fsl_udc {
++ struct device *dev;
+ struct usb_gadget gadget;
+ struct usb_gadget_driver *driver;
+ struct fsl_usb2_platform_data *pdata;
+diff --git a/drivers/usb/gadget/udc/omap_udc.c b/drivers/usb/gadget/udc/omap_udc.c
+index a1493977517e76..4b4dce8ab7dd1f 100644
+--- a/drivers/usb/gadget/udc/omap_udc.c
++++ b/drivers/usb/gadget/udc/omap_udc.c
+@@ -18,6 +18,7 @@
+ #include <linux/errno.h>
+ #include <linux/delay.h>
+ #include <linux/slab.h>
++#include <linux/string_choices.h>
+ #include <linux/timer.h>
+ #include <linux/list.h>
+ #include <linux/interrupt.h>
+@@ -1248,7 +1249,7 @@ static int omap_vbus_session(struct usb_gadget *gadget, int is_active)
+
+ udc = container_of(gadget, struct omap_udc, gadget);
+ spin_lock_irqsave(&udc->lock, flags);
+- VDBG("VBUS %s\n", is_active ? "on" : "off");
++ VDBG("VBUS %s\n", str_on_off(is_active));
+ udc->vbus_active = (is_active != 0);
+ if (cpu_is_omap15xx()) {
+ /* "software" detect, ignored if !VBUS_MODE_1510 */
+diff --git a/drivers/usb/gadget/udc/pxa27x_udc.c b/drivers/usb/gadget/udc/pxa27x_udc.c
+index 1a6317e4b2a323..5f0c7748d4b353 100644
+--- a/drivers/usb/gadget/udc/pxa27x_udc.c
++++ b/drivers/usb/gadget/udc/pxa27x_udc.c
+@@ -20,6 +20,7 @@
+ #include <linux/gpio.h>
+ #include <linux/gpio/consumer.h>
+ #include <linux/slab.h>
++#include <linux/string_choices.h>
+ #include <linux/prefetch.h>
+ #include <linux/byteorder/generic.h>
+ #include <linux/platform_data/pxa2xx_udc.h>
+@@ -1083,7 +1084,7 @@ static int pxa_ep_queue(struct usb_ep *_ep, struct usb_request *_req,
+
+ is_first_req = list_empty(&ep->queue);
+ ep_dbg(ep, "queue req %p(first=%s), len %d buf %p\n",
+- _req, is_first_req ? "yes" : "no",
++ _req, str_yes_no(is_first_req),
+ _req->length, _req->buf);
+
+ if (!ep->enabled) {
+diff --git a/drivers/usb/gadget/udc/snps_udc_core.c b/drivers/usb/gadget/udc/snps_udc_core.c
+index cd89532adec262..dd0f7125b02183 100644
+--- a/drivers/usb/gadget/udc/snps_udc_core.c
++++ b/drivers/usb/gadget/udc/snps_udc_core.c
+@@ -3133,7 +3133,6 @@ int udc_probe(struct udc *dev)
+ /* device struct setup */
+ dev->gadget.ops = &udc_ops;
+
+- dev_set_name(&dev->gadget.dev, "gadget");
+ dev->gadget.name = name;
+ dev->gadget.max_speed = USB_SPEED_HIGH;
+
+diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c
+index 5f60528453e917..bd33cf6769cf6d 100644
+--- a/drivers/usb/host/xhci-pci.c
++++ b/drivers/usb/host/xhci-pci.c
+@@ -454,6 +454,7 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
+ if (pdev->vendor == PCI_VENDOR_ID_VIA && pdev->device == PCI_DEVICE_ID_VIA_VL805) {
+ xhci->quirks |= XHCI_LPM_SUPPORT;
+ xhci->quirks |= XHCI_TRB_OVERFETCH;
++ xhci->dma_mask_bits = 36;
+ }
+
+ if (pdev->vendor == PCI_VENDOR_ID_ASMEDIA &&
+diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
+index aae3f1d1f11950..a50e813dda13a0 100644
+--- a/drivers/usb/host/xhci.c
++++ b/drivers/usb/host/xhci.c
+@@ -5259,6 +5259,7 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
+ if (xhci->hci_version > 0x100)
+ xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
+
++ xhci->dma_mask_bits = 64;
+ /* xhci-plat or xhci-pci might have set max_interrupters already */
+ if ((!xhci->max_interrupters) ||
+ xhci->max_interrupters > HCS_MAX_INTRS(xhci->hcs_params1))
+@@ -5300,12 +5301,16 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
+ if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
+ xhci->hcc_params &= ~BIT(0);
+
+- /* Set dma_mask and coherent_dma_mask to 64-bits,
+- * if xHC supports 64-bit addressing */
++ /*
++ * Set dma_mask and coherent_dma_mask to 64-bits if xHC supports
++ * 64-bit addressing, unless a controller-specific quirk callback
++ * limits the usable address width.
++ */
+ if (HCC_64BIT_ADDR(xhci->hcc_params) &&
+- !dma_set_mask(dev, DMA_BIT_MASK(64))) {
+- xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
+- dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
++ !dma_set_mask(dev, DMA_BIT_MASK(xhci->dma_mask_bits))) {
++ xhci_dbg(xhci, "Enabling %u-bit DMA addresses.\n",
++ xhci->dma_mask_bits);
++ dma_set_coherent_mask(dev, DMA_BIT_MASK(xhci->dma_mask_bits));
+ } else {
+ /*
+ * This is to avoid error in cases where a 32-bit USB
+diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
+index b4fa8e7e437639..1a3f2dfd5a711d 100644
+--- a/drivers/usb/host/xhci.h
++++ b/drivers/usb/host/xhci.h
+@@ -1510,6 +1510,7 @@ struct xhci_hcd {
+ /* imod_interval in ns (I * 250ns) */
+ u32 imod_interval;
+ u32 page_size;
++ unsigned int dma_mask_bits;
+ /* MSI-X/MSI vectors */
+ int nvecs;
+ /* optional clocks */
+diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c
+index 22d91f9f138e09..d79169d37b7266 100644
+--- a/drivers/usb/musb/omap2430.c
++++ b/drivers/usb/musb/omap2430.c
+@@ -318,13 +318,11 @@ static int omap2430_probe(struct platform_device *pdev)
+
+ glue = devm_kzalloc(&pdev->dev, sizeof(*glue), GFP_KERNEL);
+ if (!glue)
+- goto err0;
++ return -ENOMEM;
+
+ musb = platform_device_alloc("musb-hdrc", PLATFORM_DEVID_AUTO);
+- if (!musb) {
+- dev_err(&pdev->dev, "failed to allocate musb device\n");
+- goto err0;
+- }
++ if (!musb)
++ return -ENOMEM;
+
+ musb->dev.parent = &pdev->dev;
+ musb->dev.dma_mask = &omap2430_dmamask;
+@@ -348,15 +346,15 @@ static int omap2430_probe(struct platform_device *pdev)
+
+ pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+ if (!pdata)
+- goto err2;
++ goto err_put_musb;
+
+ data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+ if (!data)
+- goto err2;
++ goto err_put_musb;
+
+ config = devm_kzalloc(&pdev->dev, sizeof(*config), GFP_KERNEL);
+ if (!config)
+- goto err2;
++ goto err_put_musb;
+
+ of_property_read_u32(np, "mode", (u32 *)&pdata->mode);
+ of_property_read_u32(np, "interface-type",
+@@ -379,7 +377,7 @@ static int omap2430_probe(struct platform_device *pdev)
+ if (!control_pdev) {
+ dev_err(&pdev->dev, "Failed to get control device\n");
+ ret = -EINVAL;
+- goto err2;
++ goto err_put_musb;
+ }
+ glue->control_otghs = &control_pdev->dev;
+ }
+@@ -455,22 +453,19 @@ static int omap2430_probe(struct platform_device *pdev)
+ ret = platform_device_add(musb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register musb device\n");
+- goto err3;
++ goto err_disable_rpm;
+ }
+- of_node_put(np);
+
+ return 0;
+
+-err3:
++err_disable_rpm:
+ pm_runtime_disable(glue->dev);
+ err_put_control_otghs:
+ if (!IS_ERR(glue->control_otghs))
+ put_device(glue->control_otghs);
+-err2:
+- of_node_put(np);
++err_put_musb:
+ platform_device_put(musb);
+
+-err0:
+ return ret;
+ }
+
+diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c
+index 1824ea802b83ed..c72f7716e3ff83 100644
+--- a/drivers/usb/serial/ftdi_sio.c
++++ b/drivers/usb/serial/ftdi_sio.c
+@@ -1075,6 +1075,8 @@ static const struct usb_device_id id_table_combined[] = {
+ { USB_DEVICE_INTERFACE_NUMBER(ALTERA_VID, ALTERA_UB3_602E_PID, 3) },
+ /* Abacus Electrics */
+ { USB_DEVICE(FTDI_VID, ABACUS_OPTICAL_PROBE_PID) },
++ /* Endress+Hauser AG devices */
++ { USB_DEVICE(FTDI_VID, FTDI_EH_FXA291_PID) },
+ { } /* Terminating entry */
+ };
+
+diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h
+index 6c76cfebfd0e42..9c83c17853c871 100644
+--- a/drivers/usb/serial/ftdi_sio_ids.h
++++ b/drivers/usb/serial/ftdi_sio_ids.h
+@@ -313,6 +313,11 @@
+ #define FTDI_ELV_UDF77_PID 0xFB5E /* USB DCF Funkuhr (UDF 77) */
+ #define FTDI_ELV_UIO88_PID 0xFB5F /* USB-I/O Interface (UIO 88) */
+
++/*
++ * Endress+Hauser AG product ids (FTDI_VID)
++ */
++#define FTDI_EH_FXA291_PID 0xE510
++
+ /*
+ * EVER Eco Pro UPS (http://www.ever.com.pl/)
+ */
+diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c
+index 9406a13806724e..8cb1436e06b925 100644
+--- a/drivers/usb/serial/io_edgeport.c
++++ b/drivers/usb/serial/io_edgeport.c
+@@ -646,7 +646,8 @@ static void edge_interrupt_callback(struct urb *urb)
+ if (edge_port && edge_port->open) {
+ spin_lock_irqsave(&edge_port->ep_lock,
+ flags);
+- edge_port->txCredits += txCredits;
++ edge_port->txCredits = min(edge_port->txCredits + txCredits,
++ edge_port->maxTxCredits);
+ spin_unlock_irqrestore(&edge_port->ep_lock,
+ flags);
+ dev_dbg(dev, "%s - txcredits for port%d = %d\n",
+diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c
+index f800bf5ac1de95..e500ffd150854a 100644
+--- a/drivers/usb/serial/keyspan_pda.c
++++ b/drivers/usb/serial/keyspan_pda.c
+@@ -35,6 +35,8 @@ struct keyspan_pda_private {
+ struct work_struct unthrottle_work;
+ struct usb_serial *serial;
+ struct usb_serial_port *port;
++ bool throttled;
++ bool throttle_req;
+ };
+
+ static int keyspan_pda_write_start(struct usb_serial_port *port);
+@@ -150,6 +152,7 @@ static void keyspan_pda_rx_interrupt(struct urb *urb)
+ int retval;
+ int status = urb->status;
+ struct keyspan_pda_private *priv;
++ bool throttled = false;
+ unsigned long flags;
+
+ priv = usb_get_serial_port_data(port);
+@@ -211,16 +214,24 @@ static void keyspan_pda_rx_interrupt(struct urb *urb)
+ }
+
+ exit:
+- retval = usb_submit_urb(urb, GFP_ATOMIC);
+- if (retval)
+- dev_err(&port->dev,
+- "%s - usb_submit_urb failed with result %d\n",
+- __func__, retval);
++ spin_lock_irqsave(&port->lock, flags);
++ if (priv->throttle_req) {
++ priv->throttled = true;
++ throttled = true;
++ }
++ spin_unlock_irqrestore(&port->lock, flags);
++
++ if (!throttled) {
++ retval = usb_submit_urb(urb, GFP_ATOMIC);
++ if (retval)
++ dev_err(&port->dev, "failed to resubmit in urb: %d\n", retval);
++ }
+ }
+
+ static void keyspan_pda_rx_throttle(struct tty_struct *tty)
+ {
+ struct usb_serial_port *port = tty->driver_data;
++ struct keyspan_pda_private *priv = usb_get_serial_port_data(port);
+
+ /*
+ * Stop receiving characters. We just turn off the URB request, and
+@@ -230,16 +241,29 @@ static void keyspan_pda_rx_throttle(struct tty_struct *tty)
+ * send an XOFF, although it might make sense to foist that off upon
+ * the device too.
+ */
+- usb_kill_urb(port->interrupt_in_urb);
++ spin_lock_irq(&port->lock);
++ priv->throttle_req = true;
++ spin_unlock_irq(&port->lock);
+ }
+
+ static void keyspan_pda_rx_unthrottle(struct tty_struct *tty)
+ {
+ struct usb_serial_port *port = tty->driver_data;
++ struct keyspan_pda_private *priv = usb_get_serial_port_data(port);
++ bool throttled;
++ int ret;
+
+- /* just restart the receive interrupt URB */
+- if (usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL))
+- dev_dbg(&port->dev, "usb_submit_urb(read urb) failed\n");
++ spin_lock_irq(&port->lock);
++ throttled = priv->throttled;
++ priv->throttled = false;
++ priv->throttle_req = false;
++ spin_unlock_irq(&port->lock);
++
++ if (throttled) {
++ ret = usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL);
++ if (ret)
++ dev_err(&port->dev, "failed to submit in urb: %d\n", ret);
++ }
+ }
+
+ static speed_t keyspan_pda_setbaud(struct usb_serial *serial, speed_t baud)
+@@ -579,6 +603,8 @@ static int keyspan_pda_open(struct tty_struct *tty,
+
+ spin_lock_irq(&port->lock);
+ priv->tx_room = rc;
++ priv->throttled = false;
++ priv->throttle_req = false;
+ spin_unlock_irq(&port->lock);
+
+ rc = usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL);
+diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
+index b4b58e48e2253d..89624f38fec2eb 100644
+--- a/drivers/usb/serial/option.c
++++ b/drivers/usb/serial/option.c
+@@ -2497,6 +2497,7 @@ static const struct usb_device_id option_ids[] = {
+ .driver_info = RSVD(5) },
+ { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff), /* Rolling RW135R-GL (laptop MBIM) */
+ .driver_info = RSVD(5) },
++ { USB_DEVICE_INTERFACE_CLASS(0x3466, 0x3301, 0xff) }, /* TDTECH MT5710-CN */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) }, /* NetPrisma LCUK54-WWD for Global */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0x00, 0x40) },
+ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x40) },
+diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h
+index 255968f9ca42ae..ac22fa31873439 100644
+--- a/drivers/usb/storage/unusual_devs.h
++++ b/drivers/usb/storage/unusual_devs.h
+@@ -395,6 +395,13 @@ UNUSUAL_DEV( 0x04b3, 0x4001, 0x0110, 0x0110,
+ USB_SC_DEVICE, USB_PR_CB, NULL,
+ US_FL_MAX_SECTORS_MIN),
+
++/* Reported by Ai Chao <aichao-UOlijcLmZ/[email protected]> */
++UNUSUAL_DEV( 0x04b4, 0xb708, 0x0000, 0xffff,
++ "Longmai Technologies",
++ "USB Key",
++ USB_SC_SCSI, USB_PR_BULK, NULL,
++ US_FL_NO_ATA_1X),
++
+ /*
+ * Reported by Simon Levitt <simon-V/[email protected]>
+ * This entry needs Sub and Proto fields
+diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c
+index 58116f89d8dae9..ec743bed361c17 100644
+--- a/drivers/vdpa/vdpa_user/iova_domain.c
++++ b/drivers/vdpa/vdpa_user/iova_domain.c
+@@ -103,19 +103,38 @@ void vduse_domain_clear_map(struct vduse_iova_domain *domain,
+ static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,
+ u64 iova, u64 size, u64 paddr)
+ {
+- struct vduse_bounce_map *map;
++ struct vduse_bounce_map *map, *head_map;
++ struct page *tmp_page;
+ u64 last = iova + size - 1;
+
+ while (iova <= last) {
+- map = &domain->bounce_maps[iova >> PAGE_SHIFT];
++ /*
++ * When PAGE_SIZE is larger than 4KB, multiple adjacent bounce_maps will
++ * point to the same memory page of PAGE_SIZE. Since bounce_maps originate
++ * from IO requests, we may not be able to guarantee that the orig_phys
++ * values of all IO requests within the same 64KB memory page are contiguous.
++ * Therefore, we need to store them separately.
++ *
++ * Bounce pages are allocated on demand. As a result, it may occur that
++ * multiple bounce pages corresponding to the same 64KB memory page attempt
++ * to allocate memory simultaneously, so we use cmpxchg to handle this
++ * concurrency.
++ */
++ map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
+ if (!map->bounce_page) {
+- map->bounce_page = alloc_page(GFP_ATOMIC);
+- if (!map->bounce_page)
+- return -ENOMEM;
++ head_map = &domain->bounce_maps[(iova & PAGE_MASK) >> BOUNCE_MAP_SHIFT];
++ if (!head_map->bounce_page) {
++ tmp_page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
++ if (!tmp_page)
++ return -ENOMEM;
++ if (cmpxchg(&head_map->bounce_page, NULL, tmp_page))
++ __free_page(tmp_page);
++ }
++ map->bounce_page = head_map->bounce_page;
+ }
+ map->orig_phys = paddr;
+- paddr += PAGE_SIZE;
+- iova += PAGE_SIZE;
++ paddr += BOUNCE_MAP_SIZE;
++ iova += BOUNCE_MAP_SIZE;
+ }
+ return 0;
+ }
+@@ -127,12 +146,17 @@ static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,
+ u64 last = iova + size - 1;
+
+ while (iova <= last) {
+- map = &domain->bounce_maps[iova >> PAGE_SHIFT];
++ map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
+ map->orig_phys = INVALID_PHYS_ADDR;
+- iova += PAGE_SIZE;
++ iova += BOUNCE_MAP_SIZE;
+ }
+ }
+
++static unsigned int offset_in_bounce_page(dma_addr_t addr)
++{
++ return (addr & ~BOUNCE_MAP_MASK);
++}
++
+ static void do_bounce(phys_addr_t orig, void *addr, size_t size,
+ enum dma_data_direction dir)
+ {
+@@ -163,7 +187,7 @@ static void vduse_domain_bounce(struct vduse_iova_domain *domain,
+ {
+ struct vduse_bounce_map *map;
+ struct page *page;
+- unsigned int offset;
++ unsigned int offset, head_offset;
+ void *addr;
+ size_t sz;
+
+@@ -171,9 +195,10 @@ static void vduse_domain_bounce(struct vduse_iova_domain *domain,
+ return;
+
+ while (size) {
+- map = &domain->bounce_maps[iova >> PAGE_SHIFT];
+- offset = offset_in_page(iova);
+- sz = min_t(size_t, PAGE_SIZE - offset, size);
++ map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
++ head_offset = offset_in_page(iova);
++ offset = offset_in_bounce_page(iova);
++ sz = min_t(size_t, BOUNCE_MAP_SIZE - offset, size);
+
+ if (WARN_ON(!map->bounce_page ||
+ map->orig_phys == INVALID_PHYS_ADDR))
+@@ -183,7 +208,7 @@ static void vduse_domain_bounce(struct vduse_iova_domain *domain,
+ map->user_bounce_page : map->bounce_page;
+
+ addr = kmap_local_page(page);
+- do_bounce(map->orig_phys + offset, addr + offset, sz, dir);
++ do_bounce(map->orig_phys + offset, addr + head_offset, sz, dir);
+ kunmap_local(addr);
+ size -= sz;
+ iova += sz;
+@@ -218,7 +243,7 @@ vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)
+ struct page *page = NULL;
+
+ read_lock(&domain->bounce_lock);
+- map = &domain->bounce_maps[iova >> PAGE_SHIFT];
++ map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
+ if (domain->user_bounce_pages || !map->bounce_page)
+ goto out;
+
+@@ -236,7 +261,7 @@ vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
+ struct vduse_bounce_map *map;
+ unsigned long pfn, bounce_pfns;
+
+- bounce_pfns = domain->bounce_size >> PAGE_SHIFT;
++ bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT;
+
+ for (pfn = 0; pfn < bounce_pfns; pfn++) {
+ map = &domain->bounce_maps[pfn];
+@@ -246,7 +271,8 @@ vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
+ if (!map->bounce_page)
+ continue;
+
+- __free_page(map->bounce_page);
++ if (!((pfn << BOUNCE_MAP_SHIFT) & ~PAGE_MASK))
++ __free_page(map->bounce_page);
+ map->bounce_page = NULL;
+ }
+ }
+@@ -254,8 +280,12 @@ vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
+ int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
+ struct page **pages, int count)
+ {
+- struct vduse_bounce_map *map;
+- int i, ret;
++ struct vduse_bounce_map *map, *head_map;
++ int i, j, ret;
++ int inner_pages = PAGE_SIZE / BOUNCE_MAP_SIZE;
++ int bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT;
++ struct page *head_page = NULL;
++ bool need_copy;
+
+ /* Now we don't support partial mapping */
+ if (count != (domain->bounce_size >> PAGE_SHIFT))
+@@ -267,16 +297,23 @@ int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
+ goto out;
+
+ for (i = 0; i < count; i++) {
+- map = &domain->bounce_maps[i];
+- if (map->bounce_page) {
++ need_copy = false;
++ head_map = &domain->bounce_maps[(i * inner_pages)];
++ head_page = head_map->bounce_page;
++ for (j = 0; j < inner_pages; j++) {
++ if ((i * inner_pages + j) >= bounce_pfns)
++ break;
++ map = &domain->bounce_maps[(i * inner_pages + j)];
+ /* Copy kernel page to user page if it's in use */
+- if (map->orig_phys != INVALID_PHYS_ADDR)
+- memcpy_to_page(pages[i], 0,
+- page_address(map->bounce_page),
+- PAGE_SIZE);
++ if ((head_page) && (map->orig_phys != INVALID_PHYS_ADDR))
++ need_copy = true;
++ map->user_bounce_page = pages[i];
+ }
+- map->user_bounce_page = pages[i];
+ get_page(pages[i]);
++ if ((head_page) && (need_copy))
++ memcpy_to_page(pages[i], 0,
++ page_address(head_page),
++ PAGE_SIZE);
+ }
+ domain->user_bounce_pages = true;
+ ret = 0;
+@@ -288,8 +325,12 @@ out:
+
+ void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)
+ {
+- struct vduse_bounce_map *map;
+- unsigned long i, count;
++ struct vduse_bounce_map *map, *head_map;
++ unsigned long i, j, count;
++ int inner_pages = PAGE_SIZE / BOUNCE_MAP_SIZE;
++ int bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT;
++ struct page *head_page = NULL;
++ bool need_copy;
+
+ write_lock(&domain->bounce_lock);
+ if (!domain->user_bounce_pages)
+@@ -297,20 +338,27 @@ void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)
+
+ count = domain->bounce_size >> PAGE_SHIFT;
+ for (i = 0; i < count; i++) {
+- struct page *page = NULL;
+-
+- map = &domain->bounce_maps[i];
+- if (WARN_ON(!map->user_bounce_page))
++ need_copy = false;
++ head_map = &domain->bounce_maps[(i * inner_pages)];
++ if (WARN_ON(!head_map->user_bounce_page))
+ continue;
+-
+- /* Copy user page to kernel page if it's in use */
+- if (map->orig_phys != INVALID_PHYS_ADDR) {
+- page = map->bounce_page;
+- memcpy_from_page(page_address(page),
+- map->user_bounce_page, 0, PAGE_SIZE);
++ head_page = head_map->user_bounce_page;
++
++ for (j = 0; j < inner_pages; j++) {
++ if ((i * inner_pages + j) >= bounce_pfns)
++ break;
++ map = &domain->bounce_maps[(i * inner_pages + j)];
++ if (WARN_ON(!map->user_bounce_page))
++ continue;
++ /* Copy user page to kernel page if it's in use */
++ if ((map->orig_phys != INVALID_PHYS_ADDR) && (head_map->bounce_page))
++ need_copy = true;
++ map->user_bounce_page = NULL;
+ }
+- put_page(map->user_bounce_page);
+- map->user_bounce_page = NULL;
++ if (need_copy)
++ memcpy_from_page(page_address(head_map->bounce_page),
++ head_page, 0, PAGE_SIZE);
++ put_page(head_page);
+ }
+ domain->user_bounce_pages = false;
+ out:
+@@ -445,17 +493,15 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
+ vduse_domain_free_iova(iovad, dma_addr, size);
+ }
+
+-void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
+- size_t size, dma_addr_t *dma_addr,
+- gfp_t flag, unsigned long attrs)
++dma_addr_t vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
++ size_t size, void *orig)
+ {
+ struct iova_domain *iovad = &domain->consistent_iovad;
+ unsigned long limit = domain->iova_limit;
+ dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
+- void *orig = alloc_pages_exact(size, flag);
+
+- if (!iova || !orig)
+- goto err;
++ if (!iova)
++ return DMA_MAPPING_ERROR;
+
+ spin_lock(&domain->iotlb_lock);
+ if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,
+@@ -466,27 +512,20 @@ void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
+ }
+ spin_unlock(&domain->iotlb_lock);
+
+- *dma_addr = iova;
++ return iova;
+
+- return orig;
+ err:
+- *dma_addr = DMA_MAPPING_ERROR;
+- if (orig)
+- free_pages_exact(orig, size);
+- if (iova)
+- vduse_domain_free_iova(iovad, iova, size);
++ vduse_domain_free_iova(iovad, iova, size);
+
+- return NULL;
++ return DMA_MAPPING_ERROR;
+ }
+
+ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
+- void *vaddr, dma_addr_t dma_addr,
+- unsigned long attrs)
++ dma_addr_t dma_addr, unsigned long attrs)
+ {
+ struct iova_domain *iovad = &domain->consistent_iovad;
+ struct vhost_iotlb_map *map;
+ struct vdpa_map_file *map_file;
+- phys_addr_t pa;
+
+ spin_lock(&domain->iotlb_lock);
+ map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,
+@@ -498,12 +537,10 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
+ map_file = (struct vdpa_map_file *)map->opaque;
+ fput(map_file->file);
+ kfree(map_file);
+- pa = map->addr;
+ vhost_iotlb_map_free(domain->iotlb, map);
+ spin_unlock(&domain->iotlb_lock);
+
+ vduse_domain_free_iova(iovad, dma_addr, size);
+- free_pages_exact(phys_to_virt(pa), size);
+ }
+
+ static vm_fault_t vduse_domain_mmap_fault(struct vm_fault *vmf)
+@@ -581,7 +618,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
+ unsigned long pfn, bounce_pfns;
+ int ret;
+
+- bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
++ bounce_pfns = PAGE_ALIGN(bounce_size) >> BOUNCE_MAP_SHIFT;
+ if (iova_limit <= bounce_size)
+ return NULL;
+
+@@ -613,7 +650,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
+ rwlock_init(&domain->bounce_lock);
+ spin_lock_init(&domain->iotlb_lock);
+ init_iova_domain(&domain->stream_iovad,
+- PAGE_SIZE, IOVA_START_PFN);
++ BOUNCE_MAP_SIZE, IOVA_START_PFN);
+ ret = iova_domain_init_rcaches(&domain->stream_iovad);
+ if (ret)
+ goto err_iovad_stream;
+diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h
+index 7f3f0928ec7814..ead5ab477079b6 100644
+--- a/drivers/vdpa/vdpa_user/iova_domain.h
++++ b/drivers/vdpa/vdpa_user/iova_domain.h
+@@ -19,6 +19,11 @@
+
+ #define INVALID_PHYS_ADDR (~(phys_addr_t)0)
+
++#define BOUNCE_MAP_SHIFT 12
++#define BOUNCE_MAP_SIZE (1 << BOUNCE_MAP_SHIFT)
++#define BOUNCE_MAP_MASK (~(BOUNCE_MAP_SIZE - 1))
++#define BOUNCE_MAP_ALIGN(addr) (((addr) + BOUNCE_MAP_SIZE - 1) & ~(BOUNCE_MAP_SIZE - 1))
++
+ struct vduse_bounce_map {
+ struct page *bounce_page;
+ struct page *user_bounce_page;
+@@ -62,13 +67,11 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
+ dma_addr_t dma_addr, size_t size,
+ enum dma_data_direction dir, unsigned long attrs);
+
+-void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
+- size_t size, dma_addr_t *dma_addr,
+- gfp_t flag, unsigned long attrs);
++dma_addr_t vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
++ size_t size, void *orig);
+
+ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
+- void *vaddr, dma_addr_t dma_addr,
+- unsigned long attrs);
++ dma_addr_t dma_addr, unsigned long attrs);
+
+ void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain);
+
+diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
+index 6f0b2b04958ab3..4fbfc51407a316 100644
+--- a/drivers/vdpa/vdpa_user/vduse_dev.c
++++ b/drivers/vdpa/vdpa_user/vduse_dev.c
+@@ -883,18 +883,23 @@ static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
+ {
+ struct vduse_dev *vdev = dev_to_vduse(dev);
+ struct vduse_iova_domain *domain = vdev->domain;
+- unsigned long iova;
+ void *addr;
+
+ *dma_addr = DMA_MAPPING_ERROR;
+- addr = vduse_domain_alloc_coherent(domain, size,
+- (dma_addr_t *)&iova, flag, attrs);
++
++ addr = alloc_pages_exact(size, flag | __GFP_ZERO);
+ if (!addr)
+ return NULL;
+
+- *dma_addr = (dma_addr_t)iova;
++ *dma_addr = vduse_domain_alloc_coherent(domain, size, addr);
++ if (*dma_addr == DMA_MAPPING_ERROR)
++ goto err;
+
+ return addr;
++
++err:
++ free_pages_exact(addr, size);
++ return NULL;
+ }
+
+ static void vduse_dev_free_coherent(struct device *dev, size_t size,
+@@ -904,7 +909,8 @@ static void vduse_dev_free_coherent(struct device *dev, size_t size,
+ struct vduse_dev *vdev = dev_to_vduse(dev);
+ struct vduse_iova_domain *domain = vdev->domain;
+
+- vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
++ vduse_domain_free_coherent(domain, size, dma_addr, attrs);
++ free_pages_exact(vaddr, size);
+ }
+
+ static size_t vduse_dev_max_mapping_size(struct device *dev)
+diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig
+index 44c9ef1435a2d2..c03c455dbdebbc 100644
+--- a/drivers/video/Kconfig
++++ b/drivers/video/Kconfig
+@@ -37,6 +37,7 @@ source "drivers/char/agp/Kconfig"
+
+ source "drivers/gpu/vga/Kconfig"
+
++source "drivers/gpu/Kconfig"
+ source "drivers/gpu/host1x/Kconfig"
+ source "drivers/gpu/ipu-v3/Kconfig"
+
+diff --git a/drivers/video/fbdev/core/bitblit.c b/drivers/video/fbdev/core/bitblit.c
+index dc5ad3fcc7be4b..bed8ba18222b92 100644
+--- a/drivers/video/fbdev/core/bitblit.c
++++ b/drivers/video/fbdev/core/bitblit.c
+@@ -261,10 +261,10 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ int fg, int bg)
+ {
+ struct fb_cursor cursor;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ unsigned short charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ int w = DIV_ROUND_UP(vc->vc_font.width, 8), c;
+- int y = real_y(ops->p, vc->state.y);
++ int y = real_y(par->p, vc->state.y);
+ int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
+ int err = 1;
+ char *src;
+@@ -278,10 +278,10 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ attribute = get_attribute(info, c);
+ src = vc->vc_font.data + ((c & charmask) * (w * vc->vc_font.height));
+
+- if (ops->cursor_state.image.data != src ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.data = src;
+- cursor.set |= FB_CUR_SETIMAGE;
++ if (par->cursor_state.image.data != src ||
++ par->cursor_reset) {
++ par->cursor_state.image.data = src;
++ cursor.set |= FB_CUR_SETIMAGE;
+ }
+
+ if (attribute) {
+@@ -290,46 +290,46 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ dst = kmalloc_array(w, vc->vc_font.height, GFP_ATOMIC);
+ if (!dst)
+ return;
+- kfree(ops->cursor_data);
+- ops->cursor_data = dst;
++ kfree(par->cursor_data);
++ par->cursor_data = dst;
+ update_attr(dst, src, attribute, vc);
+ src = dst;
+ }
+
+- if (ops->cursor_state.image.fg_color != fg ||
+- ops->cursor_state.image.bg_color != bg ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.fg_color = fg;
+- ops->cursor_state.image.bg_color = bg;
++ if (par->cursor_state.image.fg_color != fg ||
++ par->cursor_state.image.bg_color != bg ||
++ par->cursor_reset) {
++ par->cursor_state.image.fg_color = fg;
++ par->cursor_state.image.bg_color = bg;
+ cursor.set |= FB_CUR_SETCMAP;
+ }
+
+- if ((ops->cursor_state.image.dx != (vc->vc_font.width * vc->state.x)) ||
+- (ops->cursor_state.image.dy != (vc->vc_font.height * y)) ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.dx = vc->vc_font.width * vc->state.x;
+- ops->cursor_state.image.dy = vc->vc_font.height * y;
++ if ((par->cursor_state.image.dx != (vc->vc_font.width * vc->state.x)) ||
++ (par->cursor_state.image.dy != (vc->vc_font.height * y)) ||
++ par->cursor_reset) {
++ par->cursor_state.image.dx = vc->vc_font.width * vc->state.x;
++ par->cursor_state.image.dy = vc->vc_font.height * y;
+ cursor.set |= FB_CUR_SETPOS;
+ }
+
+- if (ops->cursor_state.image.height != vc->vc_font.height ||
+- ops->cursor_state.image.width != vc->vc_font.width ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.height = vc->vc_font.height;
+- ops->cursor_state.image.width = vc->vc_font.width;
++ if (par->cursor_state.image.height != vc->vc_font.height ||
++ par->cursor_state.image.width != vc->vc_font.width ||
++ par->cursor_reset) {
++ par->cursor_state.image.height = vc->vc_font.height;
++ par->cursor_state.image.width = vc->vc_font.width;
+ cursor.set |= FB_CUR_SETSIZE;
+ }
+
+- if (ops->cursor_state.hot.x || ops->cursor_state.hot.y ||
+- ops->cursor_reset) {
+- ops->cursor_state.hot.x = cursor.hot.y = 0;
++ if (par->cursor_state.hot.x || par->cursor_state.hot.y ||
++ par->cursor_reset) {
++ par->cursor_state.hot.x = cursor.hot.y = 0;
+ cursor.set |= FB_CUR_SETHOT;
+ }
+
+ if (cursor.set & FB_CUR_SETSIZE ||
+- vc->vc_cursor_type != ops->p->cursor_shape ||
+- ops->cursor_state.mask == NULL ||
+- ops->cursor_reset) {
++ vc->vc_cursor_type != par->p->cursor_shape ||
++ par->cursor_state.mask == NULL ||
++ par->cursor_reset) {
+ char *mask = kmalloc_array(w, vc->vc_font.height, GFP_ATOMIC);
+ int cur_height, size, i = 0;
+ u8 msk = 0xff;
+@@ -337,13 +337,13 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ if (!mask)
+ return;
+
+- kfree(ops->cursor_state.mask);
+- ops->cursor_state.mask = mask;
++ kfree(par->cursor_state.mask);
++ par->cursor_state.mask = mask;
+
+- ops->p->cursor_shape = vc->vc_cursor_type;
++ par->p->cursor_shape = vc->vc_cursor_type;
+ cursor.set |= FB_CUR_SETSHAPE;
+
+- switch (CUR_SIZE(ops->p->cursor_shape)) {
++ switch (CUR_SIZE(par->p->cursor_shape)) {
+ case CUR_NONE:
+ cur_height = 0;
+ break;
+@@ -372,19 +372,19 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ mask[i++] = msk;
+ }
+
+- ops->cursor_state.enable = enable && !use_sw;
++ par->cursor_state.enable = enable && !use_sw;
+
+ cursor.image.data = src;
+- cursor.image.fg_color = ops->cursor_state.image.fg_color;
+- cursor.image.bg_color = ops->cursor_state.image.bg_color;
+- cursor.image.dx = ops->cursor_state.image.dx;
+- cursor.image.dy = ops->cursor_state.image.dy;
+- cursor.image.height = ops->cursor_state.image.height;
+- cursor.image.width = ops->cursor_state.image.width;
+- cursor.hot.x = ops->cursor_state.hot.x;
+- cursor.hot.y = ops->cursor_state.hot.y;
+- cursor.mask = ops->cursor_state.mask;
+- cursor.enable = ops->cursor_state.enable;
++ cursor.image.fg_color = par->cursor_state.image.fg_color;
++ cursor.image.bg_color = par->cursor_state.image.bg_color;
++ cursor.image.dx = par->cursor_state.image.dx;
++ cursor.image.dy = par->cursor_state.image.dy;
++ cursor.image.height = par->cursor_state.image.height;
++ cursor.image.width = par->cursor_state.image.width;
++ cursor.hot.x = par->cursor_state.hot.x;
++ cursor.hot.y = par->cursor_state.hot.y;
++ cursor.mask = par->cursor_state.mask;
++ cursor.enable = par->cursor_state.enable;
+ cursor.image.depth = 1;
+ cursor.rop = ROP_XOR;
+
+@@ -394,31 +394,31 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ if (err)
+ soft_cursor(info, &cursor);
+
+- ops->cursor_reset = 0;
++ par->cursor_reset = 0;
+ }
+
+ static int bit_update_start(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int err;
+
+- err = fb_pan_display(info, &ops->var);
+- ops->var.xoffset = info->var.xoffset;
+- ops->var.yoffset = info->var.yoffset;
+- ops->var.vmode = info->var.vmode;
++ err = fb_pan_display(info, &par->var);
++ par->var.xoffset = info->var.xoffset;
++ par->var.yoffset = info->var.yoffset;
++ par->var.vmode = info->var.vmode;
+ return err;
+ }
+
+-void fbcon_set_bitops(struct fbcon_ops *ops)
++void fbcon_set_bitops(struct fbcon_par *par)
+ {
+- ops->bmove = bit_bmove;
+- ops->clear = bit_clear;
+- ops->putcs = bit_putcs;
+- ops->clear_margins = bit_clear_margins;
+- ops->cursor = bit_cursor;
+- ops->update_start = bit_update_start;
+- ops->rotate_font = NULL;
+-
+- if (ops->rotate)
+- fbcon_set_rotate(ops);
++ par->bmove = bit_bmove;
++ par->clear = bit_clear;
++ par->putcs = bit_putcs;
++ par->clear_margins = bit_clear_margins;
++ par->cursor = bit_cursor;
++ par->update_start = bit_update_start;
++ par->rotate_font = NULL;
++
++ if (par->rotate)
++ fbcon_set_rotate(par);
+ }
+diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c
+index fb60d2aff5b03d..01b6a03b80b091 100644
+--- a/drivers/video/fbdev/core/fbcon.c
++++ b/drivers/video/fbdev/core/fbcon.c
+@@ -200,27 +200,27 @@ static struct device *fbcon_device;
+ #ifdef CONFIG_FRAMEBUFFER_CONSOLE_ROTATION
+ static inline void fbcon_set_rotation(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ if (!(info->flags & FBINFO_MISC_TILEBLITTING) &&
+- ops->p->con_rotate < 4)
+- ops->rotate = ops->p->con_rotate;
++ par->p->con_rotate < 4)
++ par->rotate = par->p->con_rotate;
+ else
+- ops->rotate = 0;
++ par->rotate = 0;
+ }
+
+ static void fbcon_rotate(struct fb_info *info, u32 rotate)
+ {
+- struct fbcon_ops *ops= info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fb_info *fb_info;
+
+- if (!ops || ops->currcon == -1)
++ if (!par || par->currcon == -1)
+ return;
+
+- fb_info = fbcon_info_from_console(ops->currcon);
++ fb_info = fbcon_info_from_console(par->currcon);
+
+ if (info == fb_info) {
+- struct fbcon_display *p = &fb_display[ops->currcon];
++ struct fbcon_display *p = &fb_display[par->currcon];
+
+ if (rotate < 4)
+ p->con_rotate = rotate;
+@@ -233,12 +233,12 @@ static void fbcon_rotate(struct fb_info *info, u32 rotate)
+
+ static void fbcon_rotate_all(struct fb_info *info, u32 rotate)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct vc_data *vc;
+ struct fbcon_display *p;
+ int i;
+
+- if (!ops || ops->currcon < 0 || rotate > 3)
++ if (!par || par->currcon < 0 || rotate > 3)
+ return;
+
+ for (i = first_fb_vc; i <= last_fb_vc; i++) {
+@@ -256,9 +256,9 @@ static void fbcon_rotate_all(struct fb_info *info, u32 rotate)
+ #else
+ static inline void fbcon_set_rotation(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- ops->rotate = FB_ROTATE_UR;
++ par->rotate = FB_ROTATE_UR;
+ }
+
+ static void fbcon_rotate(struct fb_info *info, u32 rotate)
+@@ -274,9 +274,9 @@ static void fbcon_rotate_all(struct fb_info *info, u32 rotate)
+
+ static int fbcon_get_rotate(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- return (ops) ? ops->rotate : 0;
++ return (par) ? par->rotate : 0;
+ }
+
+ static bool fbcon_skip_panic(struct fb_info *info)
+@@ -293,10 +293,10 @@ static bool fbcon_skip_panic(struct fb_info *info)
+
+ static inline int fbcon_is_inactive(struct vc_data *vc, struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ return (info->state != FBINFO_STATE_RUNNING ||
+- vc->vc_mode != KD_TEXT || ops->graphics || fbcon_skip_panic(info));
++ vc->vc_mode != KD_TEXT || par->graphics || fbcon_skip_panic(info));
+ }
+
+ static int get_color(struct vc_data *vc, struct fb_info *info,
+@@ -368,7 +368,7 @@ static int get_color(struct vc_data *vc, struct fb_info *info,
+
+ static void fb_flashcursor(struct work_struct *work)
+ {
+- struct fbcon_ops *ops = container_of(work, struct fbcon_ops, cursor_work.work);
++ struct fbcon_par *par = container_of(work, struct fbcon_par, cursor_work.work);
+ struct fb_info *info;
+ struct vc_data *vc = NULL;
+ int c;
+@@ -383,10 +383,10 @@ static void fb_flashcursor(struct work_struct *work)
+ return;
+
+ /* protected by console_lock */
+- info = ops->info;
++ info = par->info;
+
+- if (ops->currcon != -1)
+- vc = vc_cons[ops->currcon].d;
++ if (par->currcon != -1)
++ vc = vc_cons[par->currcon].d;
+
+ if (!vc || !con_is_visible(vc) ||
+ fbcon_info_from_console(vc->vc_num) != info ||
+@@ -396,29 +396,29 @@ static void fb_flashcursor(struct work_struct *work)
+ }
+
+ c = scr_readw((u16 *) vc->vc_pos);
+- enable = ops->cursor_flash && !ops->cursor_state.enable;
+- ops->cursor(vc, info, enable, get_color(vc, info, c, 1),
++ enable = par->cursor_flash && !par->cursor_state.enable;
++ par->cursor(vc, info, enable, get_color(vc, info, c, 1),
+ get_color(vc, info, c, 0));
+ console_unlock();
+
+- queue_delayed_work(system_power_efficient_wq, &ops->cursor_work,
+- ops->cur_blink_jiffies);
++ queue_delayed_work(system_power_efficient_wq, &par->cursor_work,
++ par->cur_blink_jiffies);
+ }
+
+ static void fbcon_add_cursor_work(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ if (!fbcon_cursor_noblink)
+- queue_delayed_work(system_power_efficient_wq, &ops->cursor_work,
+- ops->cur_blink_jiffies);
++ queue_delayed_work(system_power_efficient_wq, &par->cursor_work,
++ par->cur_blink_jiffies);
+ }
+
+ static void fbcon_del_cursor_work(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- cancel_delayed_work_sync(&ops->cursor_work);
++ cancel_delayed_work_sync(&par->cursor_work);
+ }
+
+ #ifndef MODULE
+@@ -578,7 +578,7 @@ static void fbcon_prepare_logo(struct vc_data *vc, struct fb_info *info,
+ int cols, int rows, int new_cols, int new_rows)
+ {
+ /* Need to make room for the logo */
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int cnt, erase = vc->vc_video_erase_char, step;
+ unsigned short *save = NULL, *r, *q;
+ int logo_height;
+@@ -594,7 +594,7 @@ static void fbcon_prepare_logo(struct vc_data *vc, struct fb_info *info,
+ */
+ if (fb_get_color_depth(&info->var, &info->fix) == 1)
+ erase &= ~0x400;
+- logo_height = fb_prepare_logo(info, ops->rotate);
++ logo_height = fb_prepare_logo(info, par->rotate);
+ logo_lines = DIV_ROUND_UP(logo_height, vc->vc_font.height);
+ q = (unsigned short *) (vc->vc_origin +
+ vc->vc_size_row * rows);
+@@ -666,15 +666,15 @@ static void fbcon_prepare_logo(struct vc_data *vc, struct fb_info *info,
+ #ifdef CONFIG_FB_TILEBLITTING
+ static void set_blitting_type(struct vc_data *vc, struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- ops->p = &fb_display[vc->vc_num];
++ par->p = &fb_display[vc->vc_num];
+
+ if ((info->flags & FBINFO_MISC_TILEBLITTING))
+ fbcon_set_tileops(vc, info);
+ else {
+ fbcon_set_rotation(info);
+- fbcon_set_bitops(ops);
++ fbcon_set_bitops(par);
+ }
+ }
+
+@@ -691,12 +691,12 @@ static int fbcon_invalid_charcount(struct fb_info *info, unsigned charcount)
+ #else
+ static void set_blitting_type(struct vc_data *vc, struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ info->flags &= ~FBINFO_MISC_TILEBLITTING;
+- ops->p = &fb_display[vc->vc_num];
++ par->p = &fb_display[vc->vc_num];
+ fbcon_set_rotation(info);
+- fbcon_set_bitops(ops);
++ fbcon_set_bitops(par);
+ }
+
+ static int fbcon_invalid_charcount(struct fb_info *info, unsigned charcount)
+@@ -716,13 +716,13 @@ static void fbcon_release(struct fb_info *info)
+ module_put(info->fbops->owner);
+
+ if (info->fbcon_par) {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ fbcon_del_cursor_work(info);
+- kfree(ops->cursor_state.mask);
+- kfree(ops->cursor_data);
+- kfree(ops->cursor_src);
+- kfree(ops->fontbuffer);
++ kfree(par->cursor_state.mask);
++ kfree(par->cursor_data);
++ kfree(par->cursor_src);
++ kfree(par->fontbuffer);
+ kfree(info->fbcon_par);
+ info->fbcon_par = NULL;
+ }
+@@ -730,7 +730,7 @@ static void fbcon_release(struct fb_info *info)
+
+ static int fbcon_open(struct fb_info *info)
+ {
+- struct fbcon_ops *ops;
++ struct fbcon_par *par;
+
+ if (!try_module_get(info->fbops->owner))
+ return -ENODEV;
+@@ -744,16 +744,16 @@ static int fbcon_open(struct fb_info *info)
+ }
+ unlock_fb_info(info);
+
+- ops = kzalloc(sizeof(struct fbcon_ops), GFP_KERNEL);
+- if (!ops) {
++ par = kzalloc(sizeof(*par), GFP_KERNEL);
++ if (!par) {
+ fbcon_release(info);
+ return -ENOMEM;
+ }
+
+- INIT_DELAYED_WORK(&ops->cursor_work, fb_flashcursor);
+- ops->info = info;
+- info->fbcon_par = ops;
+- ops->cur_blink_jiffies = HZ / 5;
++ INIT_DELAYED_WORK(&par->cursor_work, fb_flashcursor);
++ par->info = info;
++ info->fbcon_par = par;
++ par->cur_blink_jiffies = HZ / 5;
+
+ return 0;
+ }
+@@ -800,12 +800,12 @@ static void con2fb_release_oldinfo(struct vc_data *vc, struct fb_info *oldinfo,
+ static void con2fb_init_display(struct vc_data *vc, struct fb_info *info,
+ int unit, int show_logo)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int ret;
+
+- ops->currcon = fg_console;
++ par->currcon = fg_console;
+
+- if (info->fbops->fb_set_par && !ops->initialized) {
++ if (info->fbops->fb_set_par && !par->initialized) {
+ ret = info->fbops->fb_set_par(info);
+
+ if (ret)
+@@ -814,8 +814,8 @@ static void con2fb_init_display(struct vc_data *vc, struct fb_info *info,
+ "error code %d\n", ret);
+ }
+
+- ops->initialized = true;
+- ops->graphics = 0;
++ par->initialized = true;
++ par->graphics = 0;
+ fbcon_set_disp(info, &info->var, unit);
+
+ if (show_logo) {
+@@ -952,7 +952,7 @@ static const char *fbcon_startup(void)
+ struct vc_data *vc = vc_cons[fg_console].d;
+ const struct font_desc *font = NULL;
+ struct fb_info *info = NULL;
+- struct fbcon_ops *ops;
++ struct fbcon_par *par;
+ int rows, cols;
+
+ /*
+@@ -972,10 +972,10 @@ static const char *fbcon_startup(void)
+ if (fbcon_open(info))
+ return NULL;
+
+- ops = info->fbcon_par;
+- ops->currcon = -1;
+- ops->graphics = 1;
+- ops->cur_rotate = -1;
++ par = info->fbcon_par;
++ par->currcon = -1;
++ par->graphics = 1;
++ par->cur_rotate = -1;
+
+ p->con_rotate = initial_rotation;
+ if (p->con_rotate == -1)
+@@ -998,8 +998,8 @@ static const char *fbcon_startup(void)
+ vc->vc_font.charcount = font->charcount;
+ }
+
+- cols = FBCON_SWAP(ops->rotate, info->var.xres, info->var.yres);
+- rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
++ cols = FBCON_SWAP(par->rotate, info->var.xres, info->var.yres);
++ rows = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
+ cols /= vc->vc_font.width;
+ rows /= vc->vc_font.height;
+ vc_resize(vc, cols, rows);
+@@ -1017,7 +1017,7 @@ static const char *fbcon_startup(void)
+ static void fbcon_init(struct vc_data *vc, bool init)
+ {
+ struct fb_info *info;
+- struct fbcon_ops *ops;
++ struct fbcon_par *par;
+ struct vc_data **default_mode = vc->vc_display_fg;
+ struct vc_data *svc = *default_mode;
+ struct fbcon_display *t, *p = &fb_display[vc->vc_num];
+@@ -1092,8 +1092,8 @@ static void fbcon_init(struct vc_data *vc, bool init)
+ if (!*vc->uni_pagedict_loc)
+ con_copy_unimap(vc, svc);
+
+- ops = info->fbcon_par;
+- ops->cur_blink_jiffies = msecs_to_jiffies(vc->vc_cur_blink_ms);
++ par = info->fbcon_par;
++ par->cur_blink_jiffies = msecs_to_jiffies(vc->vc_cur_blink_ms);
+
+ p->con_rotate = initial_rotation;
+ if (p->con_rotate == -1)
+@@ -1105,8 +1105,8 @@ static void fbcon_init(struct vc_data *vc, bool init)
+
+ cols = vc->vc_cols;
+ rows = vc->vc_rows;
+- new_cols = FBCON_SWAP(ops->rotate, info->var.xres, info->var.yres);
+- new_rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
++ new_cols = FBCON_SWAP(par->rotate, info->var.xres, info->var.yres);
++ new_rows = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
+ new_cols /= vc->vc_font.width;
+ new_rows /= vc->vc_font.height;
+
+@@ -1118,7 +1118,7 @@ static void fbcon_init(struct vc_data *vc, bool init)
+ * We need to do it in fbcon_init() to prevent screen corruption.
+ */
+ if (con_is_visible(vc) && vc->vc_mode == KD_TEXT) {
+- if (info->fbops->fb_set_par && !ops->initialized) {
++ if (info->fbops->fb_set_par && !par->initialized) {
+ ret = info->fbops->fb_set_par(info);
+
+ if (ret)
+@@ -1127,10 +1127,10 @@ static void fbcon_init(struct vc_data *vc, bool init)
+ "error code %d\n", ret);
+ }
+
+- ops->initialized = true;
++ par->initialized = true;
+ }
+
+- ops->graphics = 0;
++ par->graphics = 0;
+
+ #ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
+ if ((info->flags & FBINFO_HWACCEL_COPYAREA) &&
+@@ -1154,12 +1154,12 @@ static void fbcon_init(struct vc_data *vc, bool init)
+ if (logo)
+ fbcon_prepare_logo(vc, info, cols, rows, new_cols, new_rows);
+
+- if (ops->rotate_font && ops->rotate_font(info, vc)) {
+- ops->rotate = FB_ROTATE_UR;
++ if (par->rotate_font && par->rotate_font(info, vc)) {
++ par->rotate = FB_ROTATE_UR;
+ set_blitting_type(vc, info);
+ }
+
+- ops->p = &fb_display[fg_console];
++ par->p = &fb_display[fg_console];
+ }
+
+ static void fbcon_free_font(struct fbcon_display *p)
+@@ -1197,7 +1197,7 @@ static void fbcon_deinit(struct vc_data *vc)
+ {
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+ struct fb_info *info;
+- struct fbcon_ops *ops;
++ struct fbcon_par *par;
+ int idx;
+
+ fbcon_free_font(p);
+@@ -1212,15 +1212,15 @@ static void fbcon_deinit(struct vc_data *vc)
+ if (!info)
+ goto finished;
+
+- ops = info->fbcon_par;
++ par = info->fbcon_par;
+
+- if (!ops)
++ if (!par)
+ goto finished;
+
+ if (con_is_visible(vc))
+ fbcon_del_cursor_work(info);
+
+- ops->initialized = false;
++ par->initialized = false;
+ finished:
+
+ fbcon_free_font(p);
+@@ -1267,7 +1267,7 @@ static void __fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx,
+ unsigned int height, unsigned int width)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int fg, bg;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+ u_int y_break;
+@@ -1282,7 +1282,7 @@ static void __fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx,
+ vc->vc_top = 0;
+ /*
+ * If the font dimensions are not an integral of the display
+- * dimensions then the ops->clear below won't end up clearing
++ * dimensions then the par->clear below won't end up clearing
+ * the margins. Call clear_margins here in case the logo
+ * bitmap stretched into the margin area.
+ */
+@@ -1296,11 +1296,10 @@ static void __fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx,
+ y_break = p->vrows - p->yscroll;
+ if (sy < y_break && sy + height - 1 >= y_break) {
+ u_int b = y_break - sy;
+- ops->clear(vc, info, real_y(p, sy), sx, b, width, fg, bg);
+- ops->clear(vc, info, real_y(p, sy + b), sx, height - b,
+- width, fg, bg);
++ par->clear(vc, info, real_y(p, sy), sx, b, width, fg, bg);
++ par->clear(vc, info, real_y(p, sy + b), sx, height - b, width, fg, bg);
+ } else
+- ops->clear(vc, info, real_y(p, sy), sx, height, width, fg, bg);
++ par->clear(vc, info, real_y(p, sy), sx, height, width, fg, bg);
+ }
+
+ static void fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx,
+@@ -1314,10 +1313,10 @@ static void fbcon_putcs(struct vc_data *vc, const u16 *s, unsigned int count,
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ if (!fbcon_is_inactive(vc, info))
+- ops->putcs(vc, info, s, count, real_y(p, ypos), xpos,
++ par->putcs(vc, info, s, count, real_y(p, ypos), xpos,
+ get_color(vc, info, scr_readw(s), 1),
+ get_color(vc, info, scr_readw(s), 0));
+ }
+@@ -1325,19 +1324,19 @@ static void fbcon_putcs(struct vc_data *vc, const u16 *s, unsigned int count,
+ static void fbcon_clear_margins(struct vc_data *vc, int bottom_only)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ if (!fbcon_is_inactive(vc, info))
+- ops->clear_margins(vc, info, margin_color, bottom_only);
++ par->clear_margins(vc, info, margin_color, bottom_only);
+ }
+
+ static void fbcon_cursor(struct vc_data *vc, bool enable)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int c = scr_readw((u16 *) vc->vc_pos);
+
+- ops->cur_blink_jiffies = msecs_to_jiffies(vc->vc_cur_blink_ms);
++ par->cur_blink_jiffies = msecs_to_jiffies(vc->vc_cur_blink_ms);
+
+ if (fbcon_is_inactive(vc, info) || vc->vc_deccm != 1)
+ return;
+@@ -1347,12 +1346,12 @@ static void fbcon_cursor(struct vc_data *vc, bool enable)
+ else
+ fbcon_add_cursor_work(info);
+
+- ops->cursor_flash = enable;
++ par->cursor_flash = enable;
+
+- if (!ops->cursor)
++ if (!par->cursor)
+ return;
+
+- ops->cursor(vc, info, enable, get_color(vc, info, c, 1),
++ par->cursor(vc, info, enable, get_color(vc, info, c, 1),
+ get_color(vc, info, c, 0));
+ }
+
+@@ -1366,9 +1365,8 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var,
+ struct fbcon_display *p, *t;
+ struct vc_data **default_mode, *vc;
+ struct vc_data *svc;
+- struct fbcon_ops *ops = info->fbcon_par;
+- int rows, cols;
+- unsigned long ret = 0;
++ struct fbcon_par *par = info->fbcon_par;
++ int rows, cols, ret;
+
+ p = &fb_display[unit];
+
+@@ -1399,7 +1397,7 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var,
+ var->yoffset = info->var.yoffset;
+ var->xoffset = info->var.xoffset;
+ fb_set_var(info, var);
+- ops->var = info->var;
++ par->var = info->var;
+ vc->vc_can_do_color = (fb_get_color_depth(&info->var, &info->fix)!=1);
+ vc->vc_complement_mask = vc->vc_can_do_color ? 0x7700 : 0x0800;
+ if (vc->vc_font.charcount == 256) {
+@@ -1415,8 +1413,8 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var,
+ if (!*vc->uni_pagedict_loc)
+ con_copy_unimap(vc, svc);
+
+- cols = FBCON_SWAP(ops->rotate, info->var.xres, info->var.yres);
+- rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
++ cols = FBCON_SWAP(par->rotate, info->var.xres, info->var.yres);
++ rows = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
+ cols /= vc->vc_font.width;
+ rows /= vc->vc_font.height;
+ ret = vc_resize(vc, cols, rows);
+@@ -1428,16 +1426,16 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var,
+ static __inline__ void ywrap_up(struct vc_data *vc, int count)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+
+ p->yscroll += count;
+ if (p->yscroll >= p->vrows) /* Deal with wrap */
+ p->yscroll -= p->vrows;
+- ops->var.xoffset = 0;
+- ops->var.yoffset = p->yscroll * vc->vc_font.height;
+- ops->var.vmode |= FB_VMODE_YWRAP;
+- ops->update_start(info);
++ par->var.xoffset = 0;
++ par->var.yoffset = p->yscroll * vc->vc_font.height;
++ par->var.vmode |= FB_VMODE_YWRAP;
++ par->update_start(info);
+ scrollback_max += count;
+ if (scrollback_max > scrollback_phys_max)
+ scrollback_max = scrollback_phys_max;
+@@ -1447,16 +1445,16 @@ static __inline__ void ywrap_up(struct vc_data *vc, int count)
+ static __inline__ void ywrap_down(struct vc_data *vc, int count)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+
+ p->yscroll -= count;
+ if (p->yscroll < 0) /* Deal with wrap */
+ p->yscroll += p->vrows;
+- ops->var.xoffset = 0;
+- ops->var.yoffset = p->yscroll * vc->vc_font.height;
+- ops->var.vmode |= FB_VMODE_YWRAP;
+- ops->update_start(info);
++ par->var.xoffset = 0;
++ par->var.yoffset = p->yscroll * vc->vc_font.height;
++ par->var.vmode |= FB_VMODE_YWRAP;
++ par->update_start(info);
+ scrollback_max -= count;
+ if (scrollback_max < 0)
+ scrollback_max = 0;
+@@ -1467,19 +1465,19 @@ static __inline__ void ypan_up(struct vc_data *vc, int count)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ p->yscroll += count;
+ if (p->yscroll > p->vrows - vc->vc_rows) {
+- ops->bmove(vc, info, p->vrows - vc->vc_rows,
++ par->bmove(vc, info, p->vrows - vc->vc_rows,
+ 0, 0, 0, vc->vc_rows, vc->vc_cols);
+ p->yscroll -= p->vrows - vc->vc_rows;
+ }
+
+- ops->var.xoffset = 0;
+- ops->var.yoffset = p->yscroll * vc->vc_font.height;
+- ops->var.vmode &= ~FB_VMODE_YWRAP;
+- ops->update_start(info);
++ par->var.xoffset = 0;
++ par->var.yoffset = p->yscroll * vc->vc_font.height;
++ par->var.vmode &= ~FB_VMODE_YWRAP;
++ par->update_start(info);
+ fbcon_clear_margins(vc, 1);
+ scrollback_max += count;
+ if (scrollback_max > scrollback_phys_max)
+@@ -1490,7 +1488,7 @@ static __inline__ void ypan_up(struct vc_data *vc, int count)
+ static __inline__ void ypan_up_redraw(struct vc_data *vc, int t, int count)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+
+ p->yscroll += count;
+@@ -1500,10 +1498,10 @@ static __inline__ void ypan_up_redraw(struct vc_data *vc, int t, int count)
+ fbcon_redraw_move(vc, p, t + count, vc->vc_rows - count, t);
+ }
+
+- ops->var.xoffset = 0;
+- ops->var.yoffset = p->yscroll * vc->vc_font.height;
+- ops->var.vmode &= ~FB_VMODE_YWRAP;
+- ops->update_start(info);
++ par->var.xoffset = 0;
++ par->var.yoffset = p->yscroll * vc->vc_font.height;
++ par->var.vmode &= ~FB_VMODE_YWRAP;
++ par->update_start(info);
+ fbcon_clear_margins(vc, 1);
+ scrollback_max += count;
+ if (scrollback_max > scrollback_phys_max)
+@@ -1515,19 +1513,19 @@ static __inline__ void ypan_down(struct vc_data *vc, int count)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ p->yscroll -= count;
+ if (p->yscroll < 0) {
+- ops->bmove(vc, info, 0, 0, p->vrows - vc->vc_rows,
++ par->bmove(vc, info, 0, 0, p->vrows - vc->vc_rows,
+ 0, vc->vc_rows, vc->vc_cols);
+ p->yscroll += p->vrows - vc->vc_rows;
+ }
+
+- ops->var.xoffset = 0;
+- ops->var.yoffset = p->yscroll * vc->vc_font.height;
+- ops->var.vmode &= ~FB_VMODE_YWRAP;
+- ops->update_start(info);
++ par->var.xoffset = 0;
++ par->var.yoffset = p->yscroll * vc->vc_font.height;
++ par->var.vmode &= ~FB_VMODE_YWRAP;
++ par->update_start(info);
+ fbcon_clear_margins(vc, 1);
+ scrollback_max -= count;
+ if (scrollback_max < 0)
+@@ -1538,7 +1536,7 @@ static __inline__ void ypan_down(struct vc_data *vc, int count)
+ static __inline__ void ypan_down_redraw(struct vc_data *vc, int t, int count)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+
+ p->yscroll -= count;
+@@ -1548,10 +1546,10 @@ static __inline__ void ypan_down_redraw(struct vc_data *vc, int t, int count)
+ fbcon_redraw_move(vc, p, t, vc->vc_rows - count, t + count);
+ }
+
+- ops->var.xoffset = 0;
+- ops->var.yoffset = p->yscroll * vc->vc_font.height;
+- ops->var.vmode &= ~FB_VMODE_YWRAP;
+- ops->update_start(info);
++ par->var.xoffset = 0;
++ par->var.yoffset = p->yscroll * vc->vc_font.height;
++ par->var.vmode &= ~FB_VMODE_YWRAP;
++ par->update_start(info);
+ fbcon_clear_margins(vc, 1);
+ scrollback_max -= count;
+ if (scrollback_max < 0)
+@@ -1600,7 +1598,7 @@ static void fbcon_redraw_blit(struct vc_data *vc, struct fb_info *info,
+ unsigned short *d = (unsigned short *)
+ (vc->vc_origin + vc->vc_size_row * line);
+ unsigned short *s = d + offset;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ while (count--) {
+ unsigned short *start = s;
+@@ -1613,8 +1611,8 @@ static void fbcon_redraw_blit(struct vc_data *vc, struct fb_info *info,
+
+ if (c == scr_readw(d)) {
+ if (s > start) {
+- ops->bmove(vc, info, line + ycount, x,
+- line, x, 1, s-start);
++ par->bmove(vc, info, line + ycount, x,
++ line, x, 1, s - start);
+ x += s - start + 1;
+ start = s + 1;
+ } else {
+@@ -1629,8 +1627,7 @@ static void fbcon_redraw_blit(struct vc_data *vc, struct fb_info *info,
+ d++;
+ } while (s < le);
+ if (s > start)
+- ops->bmove(vc, info, line + ycount, x, line, x, 1,
+- s-start);
++ par->bmove(vc, info, line + ycount, x, line, x, 1, s - start);
+ console_conditional_schedule();
+ if (ycount > 0)
+ line++;
+@@ -1701,7 +1698,7 @@ static void fbcon_bmove_rec(struct vc_data *vc, struct fbcon_display *p, int sy,
+ int dy, int dx, int height, int width, u_int y_break)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u_int b;
+
+ if (sy < y_break && sy + height > y_break) {
+@@ -1735,8 +1732,7 @@ static void fbcon_bmove_rec(struct vc_data *vc, struct fbcon_display *p, int sy,
+ }
+ return;
+ }
+- ops->bmove(vc, info, real_y(p, sy), sx, real_y(p, dy), dx,
+- height, width);
++ par->bmove(vc, info, real_y(p, sy), sx, real_y(p, dy), dx, height, width);
+ }
+
+ static void fbcon_bmove(struct vc_data *vc, int sy, int sx, int dy, int dx,
+@@ -1963,15 +1959,13 @@ static void updatescrollmode_accel(struct fbcon_display *p,
+ struct vc_data *vc)
+ {
+ #ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int cap = info->flags;
+ u16 t = 0;
+- int ypan = FBCON_SWAP(ops->rotate, info->fix.ypanstep,
+- info->fix.xpanstep);
+- int ywrap = FBCON_SWAP(ops->rotate, info->fix.ywrapstep, t);
+- int yres = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
+- int vyres = FBCON_SWAP(ops->rotate, info->var.yres_virtual,
+- info->var.xres_virtual);
++ int ypan = FBCON_SWAP(par->rotate, info->fix.ypanstep, info->fix.xpanstep);
++ int ywrap = FBCON_SWAP(par->rotate, info->fix.ywrapstep, t);
++ int yres = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
++ int vyres = FBCON_SWAP(par->rotate, info->var.yres_virtual, info->var.xres_virtual);
+ int good_pan = (cap & FBINFO_HWACCEL_YPAN) &&
+ divides(ypan, vc->vc_font.height) && vyres > yres;
+ int good_wrap = (cap & FBINFO_HWACCEL_YWRAP) &&
+@@ -2004,11 +1998,10 @@ static void updatescrollmode(struct fbcon_display *p,
+ struct fb_info *info,
+ struct vc_data *vc)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int fh = vc->vc_font.height;
+- int yres = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
+- int vyres = FBCON_SWAP(ops->rotate, info->var.yres_virtual,
+- info->var.xres_virtual);
++ int yres = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
++ int vyres = FBCON_SWAP(par->rotate, info->var.yres_virtual, info->var.xres_virtual);
+
+ p->vrows = vyres/fh;
+ if (yres > (fh * (vc->vc_rows + 1)))
+@@ -2027,7 +2020,7 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width,
+ unsigned int height, bool from_user)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+ struct fb_var_screeninfo var = info->var;
+ int x_diff, y_diff, virt_w, virt_h, virt_fw, virt_fh;
+@@ -2050,12 +2043,10 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width,
+ return -EINVAL;
+ }
+
+- virt_w = FBCON_SWAP(ops->rotate, width, height);
+- virt_h = FBCON_SWAP(ops->rotate, height, width);
+- virt_fw = FBCON_SWAP(ops->rotate, vc->vc_font.width,
+- vc->vc_font.height);
+- virt_fh = FBCON_SWAP(ops->rotate, vc->vc_font.height,
+- vc->vc_font.width);
++ virt_w = FBCON_SWAP(par->rotate, width, height);
++ virt_h = FBCON_SWAP(par->rotate, height, width);
++ virt_fw = FBCON_SWAP(par->rotate, vc->vc_font.width, vc->vc_font.height);
++ virt_fh = FBCON_SWAP(par->rotate, vc->vc_font.height, vc->vc_font.width);
+ var.xres = virt_w * virt_fw;
+ var.yres = virt_h * virt_fh;
+ x_diff = info->var.xres - var.xres;
+@@ -2081,7 +2072,7 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width,
+ fb_set_var(info, &var);
+ }
+ var_to_display(p, &info->var, info);
+- ops->var = info->var;
++ par->var = info->var;
+ }
+ updatescrollmode(p, info, vc);
+ return 0;
+@@ -2090,13 +2081,13 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width,
+ static bool fbcon_switch(struct vc_data *vc)
+ {
+ struct fb_info *info, *old_info = NULL;
+- struct fbcon_ops *ops;
++ struct fbcon_par *par;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+ struct fb_var_screeninfo var;
+ int i, ret, prev_console;
+
+ info = fbcon_info_from_console(vc->vc_num);
+- ops = info->fbcon_par;
++ par = info->fbcon_par;
+
+ if (logo_shown >= 0) {
+ struct vc_data *conp2 = vc_cons[logo_shown].d;
+@@ -2107,7 +2098,7 @@ static bool fbcon_switch(struct vc_data *vc)
+ logo_shown = FBCON_LOGO_CANSHOW;
+ }
+
+- prev_console = ops->currcon;
++ prev_console = par->currcon;
+ if (prev_console != -1)
+ old_info = fbcon_info_from_console(prev_console);
+ /*
+@@ -2120,9 +2111,9 @@ static bool fbcon_switch(struct vc_data *vc)
+ */
+ fbcon_for_each_registered_fb(i) {
+ if (fbcon_registered_fb[i]->fbcon_par) {
+- struct fbcon_ops *o = fbcon_registered_fb[i]->fbcon_par;
++ struct fbcon_par *par = fbcon_registered_fb[i]->fbcon_par;
+
+- o->currcon = vc->vc_num;
++ par->currcon = vc->vc_num;
+ }
+ }
+ memset(&var, 0, sizeof(struct fb_var_screeninfo));
+@@ -2136,7 +2127,7 @@ static bool fbcon_switch(struct vc_data *vc)
+ info->var.activate = var.activate;
+ var.vmode |= info->var.vmode & ~FB_VMODE_MASK;
+ fb_set_var(info, &var);
+- ops->var = info->var;
++ par->var = info->var;
+
+ if (old_info != NULL && (old_info != info ||
+ info->flags & FBINFO_MISC_ALWAYS_SETPAR)) {
+@@ -2154,16 +2145,16 @@ static bool fbcon_switch(struct vc_data *vc)
+ }
+
+ if (fbcon_is_inactive(vc, info) ||
+- ops->blank_state != FB_BLANK_UNBLANK)
++ par->blank_state != FB_BLANK_UNBLANK)
+ fbcon_del_cursor_work(info);
+ else
+ fbcon_add_cursor_work(info);
+
+ set_blitting_type(vc, info);
+- ops->cursor_reset = 1;
++ par->cursor_reset = 1;
+
+- if (ops->rotate_font && ops->rotate_font(info, vc)) {
+- ops->rotate = FB_ROTATE_UR;
++ if (par->rotate_font && par->rotate_font(info, vc)) {
++ par->rotate = FB_ROTATE_UR;
+ set_blitting_type(vc, info);
+ }
+
+@@ -2194,8 +2185,8 @@ static bool fbcon_switch(struct vc_data *vc)
+ scrollback_current = 0;
+
+ if (!fbcon_is_inactive(vc, info)) {
+- ops->var.xoffset = ops->var.yoffset = p->yscroll = 0;
+- ops->update_start(info);
++ par->var.xoffset = par->var.yoffset = p->yscroll = 0;
++ par->update_start(info);
+ }
+
+ fbcon_set_palette(vc, color_table);
+@@ -2204,7 +2195,7 @@ static bool fbcon_switch(struct vc_data *vc)
+ if (logo_shown == FBCON_LOGO_DRAW) {
+
+ logo_shown = fg_console;
+- fb_show_logo(info, ops->rotate);
++ fb_show_logo(info, par->rotate);
+ update_region(vc,
+ vc->vc_origin + vc->vc_size_row * vc->vc_top,
+ vc->vc_size_row * (vc->vc_bottom -
+@@ -2233,27 +2224,27 @@ static bool fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank,
+ bool mode_switch)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+ if (mode_switch) {
+ struct fb_var_screeninfo var = info->var;
+
+- ops->graphics = 1;
++ par->graphics = 1;
+
+ if (!blank) {
+ var.activate = FB_ACTIVATE_NOW | FB_ACTIVATE_FORCE |
+ FB_ACTIVATE_KD_TEXT;
+ fb_set_var(info, &var);
+- ops->graphics = 0;
+- ops->var = info->var;
++ par->graphics = 0;
++ par->var = info->var;
+ }
+ }
+
+ if (!fbcon_is_inactive(vc, info)) {
+- if (ops->blank_state != blank) {
+- ops->blank_state = blank;
++ if (par->blank_state != blank) {
++ par->blank_state = blank;
+ fbcon_cursor(vc, !blank);
+- ops->cursor_flash = (!blank);
++ par->cursor_flash = (!blank);
+
+ if (fb_blank(info, blank))
+ fbcon_generic_blank(vc, info, blank);
+@@ -2264,7 +2255,7 @@ static bool fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank,
+ }
+
+ if (mode_switch || fbcon_is_inactive(vc, info) ||
+- ops->blank_state != FB_BLANK_UNBLANK)
++ par->blank_state != FB_BLANK_UNBLANK)
+ fbcon_del_cursor_work(info);
+ else
+ fbcon_add_cursor_work(info);
+@@ -2275,10 +2266,10 @@ static bool fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank,
+ static void fbcon_debug_enter(struct vc_data *vc)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- ops->save_graphics = ops->graphics;
+- ops->graphics = 0;
++ par->save_graphics = par->graphics;
++ par->graphics = 0;
+ if (info->fbops->fb_debug_enter)
+ info->fbops->fb_debug_enter(info);
+ fbcon_set_palette(vc, color_table);
+@@ -2287,9 +2278,9 @@ static void fbcon_debug_enter(struct vc_data *vc)
+ static void fbcon_debug_leave(struct vc_data *vc)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- ops->graphics = ops->save_graphics;
++ par->graphics = par->save_graphics;
+ if (info->fbops->fb_debug_leave)
+ info->fbops->fb_debug_leave(info);
+ }
+@@ -2424,7 +2415,7 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount,
+ const u8 * data, int userfont)
+ {
+ struct fb_info *info = fbcon_info_from_console(vc->vc_num);
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fbcon_display *p = &fb_display[vc->vc_num];
+ int resize, ret, old_userfont, old_width, old_height, old_charcount;
+ u8 *old_data = vc->vc_font.data;
+@@ -2451,8 +2442,8 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount,
+ if (resize) {
+ int cols, rows;
+
+- cols = FBCON_SWAP(ops->rotate, info->var.xres, info->var.yres);
+- rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
++ cols = FBCON_SWAP(par->rotate, info->var.xres, info->var.yres);
++ rows = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
+ cols /= w;
+ rows /= h;
+ ret = vc_resize(vc, cols, rows);
+@@ -2657,11 +2648,11 @@ static void fbcon_invert_region(struct vc_data *vc, u16 * p, int cnt)
+ void fbcon_suspended(struct fb_info *info)
+ {
+ struct vc_data *vc = NULL;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- if (!ops || ops->currcon < 0)
++ if (!par || par->currcon < 0)
+ return;
+- vc = vc_cons[ops->currcon].d;
++ vc = vc_cons[par->currcon].d;
+
+ /* Clear cursor, restore saved data */
+ fbcon_cursor(vc, false);
+@@ -2670,27 +2661,27 @@ void fbcon_suspended(struct fb_info *info)
+ void fbcon_resumed(struct fb_info *info)
+ {
+ struct vc_data *vc;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- if (!ops || ops->currcon < 0)
++ if (!par || par->currcon < 0)
+ return;
+- vc = vc_cons[ops->currcon].d;
++ vc = vc_cons[par->currcon].d;
+
+ update_screen(vc);
+ }
+
+ static void fbcon_modechanged(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct vc_data *vc;
+ struct fbcon_display *p;
+ int rows, cols;
+
+- if (!ops || ops->currcon < 0)
++ if (!par || par->currcon < 0)
+ return;
+- vc = vc_cons[ops->currcon].d;
++ vc = vc_cons[par->currcon].d;
+ if (vc->vc_mode != KD_TEXT ||
+- fbcon_info_from_console(ops->currcon) != info)
++ fbcon_info_from_console(par->currcon) != info)
+ return;
+
+ p = &fb_display[vc->vc_num];
+@@ -2698,8 +2689,8 @@ static void fbcon_modechanged(struct fb_info *info)
+
+ if (con_is_visible(vc)) {
+ var_to_display(p, &info->var, info);
+- cols = FBCON_SWAP(ops->rotate, info->var.xres, info->var.yres);
+- rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
++ cols = FBCON_SWAP(par->rotate, info->var.xres, info->var.yres);
++ rows = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
+ cols /= vc->vc_font.width;
+ rows /= vc->vc_font.height;
+ vc_resize(vc, cols, rows);
+@@ -2708,8 +2699,8 @@ static void fbcon_modechanged(struct fb_info *info)
+ scrollback_current = 0;
+
+ if (!fbcon_is_inactive(vc, info)) {
+- ops->var.xoffset = ops->var.yoffset = p->yscroll = 0;
+- ops->update_start(info);
++ par->var.xoffset = par->var.yoffset = p->yscroll = 0;
++ par->update_start(info);
+ }
+
+ fbcon_set_palette(vc, color_table);
+@@ -2719,12 +2710,12 @@ static void fbcon_modechanged(struct fb_info *info)
+
+ static void fbcon_set_all_vcs(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct vc_data *vc;
+ struct fbcon_display *p;
+ int i, rows, cols, fg = -1;
+
+- if (!ops || ops->currcon < 0)
++ if (!par || par->currcon < 0)
+ return;
+
+ for (i = first_fb_vc; i <= last_fb_vc; i++) {
+@@ -2741,8 +2732,8 @@ static void fbcon_set_all_vcs(struct fb_info *info)
+ p = &fb_display[vc->vc_num];
+ set_blitting_type(vc, info);
+ var_to_display(p, &info->var, info);
+- cols = FBCON_SWAP(ops->rotate, info->var.xres, info->var.yres);
+- rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres);
++ cols = FBCON_SWAP(par->rotate, info->var.xres, info->var.yres);
++ rows = FBCON_SWAP(par->rotate, info->var.yres, info->var.xres);
+ cols /= vc->vc_font.width;
+ rows /= vc->vc_font.height;
+ vc_resize(vc, cols, rows);
+@@ -2765,13 +2756,13 @@ EXPORT_SYMBOL(fbcon_update_vcs);
+ /* let fbcon check if it supports a new screen resolution */
+ int fbcon_modechange_possible(struct fb_info *info, struct fb_var_screeninfo *var)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct vc_data *vc;
+ unsigned int i;
+
+ WARN_CONSOLE_UNLOCKED();
+
+- if (!ops)
++ if (!par)
+ return 0;
+
+ /* prevent setting a screen size which is smaller than font size */
+@@ -3069,15 +3060,14 @@ int fbcon_fb_registered(struct fb_info *info)
+
+ void fbcon_fb_blanked(struct fb_info *info, int blank)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct vc_data *vc;
+
+- if (!ops || ops->currcon < 0)
++ if (!par || par->currcon < 0)
+ return;
+
+- vc = vc_cons[ops->currcon].d;
+- if (vc->vc_mode != KD_TEXT ||
+- fbcon_info_from_console(ops->currcon) != info)
++ vc = vc_cons[par->currcon].d;
++ if (vc->vc_mode != KD_TEXT || fbcon_info_from_console(par->currcon) != info)
+ return;
+
+ if (con_is_visible(vc)) {
+@@ -3086,7 +3076,7 @@ void fbcon_fb_blanked(struct fb_info *info, int blank)
+ else
+ do_unblank_screen(0);
+ }
+- ops->blank_state = blank;
++ par->blank_state = blank;
+ }
+
+ void fbcon_new_modelist(struct fb_info *info)
+@@ -3276,7 +3266,7 @@ static ssize_t show_cursor_blink(struct device *device,
+ struct device_attribute *attr, char *buf)
+ {
+ struct fb_info *info;
+- struct fbcon_ops *ops;
++ struct fbcon_par *par;
+ int idx, blink = -1;
+
+ console_lock();
+@@ -3286,12 +3276,12 @@ static ssize_t show_cursor_blink(struct device *device,
+ goto err;
+
+ info = fbcon_registered_fb[idx];
+- ops = info->fbcon_par;
++ par = info->fbcon_par;
+
+- if (!ops)
++ if (!par)
+ goto err;
+
+- blink = delayed_work_pending(&ops->cursor_work);
++ blink = delayed_work_pending(&par->cursor_work);
+ err:
+ console_unlock();
+ return sysfs_emit(buf, "%d\n", blink);
+diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h
+index 7e21c8b3366923..b22c33e278c19b 100644
+--- a/drivers/video/fbdev/core/fbcon.h
++++ b/drivers/video/fbdev/core/fbcon.h
+@@ -50,7 +50,7 @@ struct fbcon_display {
+ const struct fb_videomode *mode;
+ };
+
+-struct fbcon_ops {
++struct fbcon_par {
+ void (*bmove)(struct vc_data *vc, struct fb_info *info, int sy,
+ int sx, int dy, int dx, int height, int width);
+ void (*clear)(struct vc_data *vc, struct fb_info *info, int sy,
+@@ -185,7 +185,7 @@ static inline u_short fb_scrollmode(struct fbcon_display *fb)
+ #ifdef CONFIG_FB_TILEBLITTING
+ extern void fbcon_set_tileops(struct vc_data *vc, struct fb_info *info);
+ #endif
+-extern void fbcon_set_bitops(struct fbcon_ops *ops);
++extern void fbcon_set_bitops(struct fbcon_par *par);
+ extern int soft_cursor(struct fb_info *info, struct fb_cursor *cursor);
+
+ #define FBCON_ATTRIBUTE_UNDERLINE 1
+@@ -224,7 +224,7 @@ static inline int get_attribute(struct fb_info *info, u16 c)
+ (i == FB_ROTATE_UR || i == FB_ROTATE_UD) ? _r : _v; })
+
+ #ifdef CONFIG_FRAMEBUFFER_CONSOLE_ROTATION
+-extern void fbcon_set_rotate(struct fbcon_ops *ops);
++extern void fbcon_set_rotate(struct fbcon_par *par);
+ #else
+ #define fbcon_set_rotate(x) do {} while(0)
+ #endif /* CONFIG_FRAMEBUFFER_CONSOLE_ROTATION */
+diff --git a/drivers/video/fbdev/core/fbcon_ccw.c b/drivers/video/fbdev/core/fbcon_ccw.c
+index 89ef4ba7e8672b..2ba8ec4c3e2bc6 100644
+--- a/drivers/video/fbdev/core/fbcon_ccw.c
++++ b/drivers/video/fbdev/core/fbcon_ccw.c
+@@ -63,9 +63,9 @@ static void ccw_update_attr(u8 *dst, u8 *src, int attribute,
+ static void ccw_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ int sx, int dy, int dx, int height, int width)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fb_copyarea area;
+- u32 vyres = GETVYRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
+
+ area.sx = sy * vc->vc_font.height;
+ area.sy = vyres - ((sx + width) * vc->vc_font.width);
+@@ -80,9 +80,9 @@ static void ccw_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ static void ccw_clear(struct vc_data *vc, struct fb_info *info, int sy,
+ int sx, int height, int width, int fg, int bg)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fb_fillrect region;
+- u32 vyres = GETVYRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
+
+ region.color = bg;
+ region.dx = sy * vc->vc_font.height;
+@@ -99,13 +99,13 @@ static inline void ccw_putcs_aligned(struct vc_data *vc, struct fb_info *info,
+ u32 d_pitch, u32 s_pitch, u32 cellsize,
+ struct fb_image *image, u8 *buf, u8 *dst)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u16 charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ u32 idx = (vc->vc_font.height + 7) >> 3;
+ u8 *src;
+
+ while (cnt--) {
+- src = ops->fontbuffer + (scr_readw(s--) & charmask)*cellsize;
++ src = par->fontbuffer + (scr_readw(s--) & charmask) * cellsize;
+
+ if (attr) {
+ ccw_update_attr(buf, src, attr, vc);
+@@ -130,7 +130,7 @@ static void ccw_putcs(struct vc_data *vc, struct fb_info *info,
+ int fg, int bg)
+ {
+ struct fb_image image;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u32 width = (vc->vc_font.height + 7)/8;
+ u32 cellsize = width * vc->vc_font.width;
+ u32 maxcnt = info->pixmap.size/cellsize;
+@@ -139,9 +139,9 @@ static void ccw_putcs(struct vc_data *vc, struct fb_info *info,
+ u32 cnt, pitch, size;
+ u32 attribute = get_attribute(info, scr_readw(s));
+ u8 *dst, *buf = NULL;
+- u32 vyres = GETVYRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
+
+- if (!ops->fontbuffer)
++ if (!par->fontbuffer)
+ return;
+
+ image.fg_color = fg;
+@@ -221,28 +221,28 @@ static void ccw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ int fg, int bg)
+ {
+ struct fb_cursor cursor;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ unsigned short charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ int w = (vc->vc_font.height + 7) >> 3, c;
+- int y = real_y(ops->p, vc->state.y);
++ int y = real_y(par->p, vc->state.y);
+ int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
+ int err = 1, dx, dy;
+ char *src;
+- u32 vyres = GETVYRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
+
+- if (!ops->fontbuffer)
++ if (!par->fontbuffer)
+ return;
+
+ cursor.set = 0;
+
+ c = scr_readw((u16 *) vc->vc_pos);
+ attribute = get_attribute(info, c);
+- src = ops->fontbuffer + ((c & charmask) * (w * vc->vc_font.width));
++ src = par->fontbuffer + ((c & charmask) * (w * vc->vc_font.width));
+
+- if (ops->cursor_state.image.data != src ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.data = src;
+- cursor.set |= FB_CUR_SETIMAGE;
++ if (par->cursor_state.image.data != src ||
++ par->cursor_reset) {
++ par->cursor_state.image.data = src;
++ cursor.set |= FB_CUR_SETIMAGE;
+ }
+
+ if (attribute) {
+@@ -251,49 +251,49 @@ static void ccw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ dst = kmalloc_array(w, vc->vc_font.width, GFP_ATOMIC);
+ if (!dst)
+ return;
+- kfree(ops->cursor_data);
+- ops->cursor_data = dst;
++ kfree(par->cursor_data);
++ par->cursor_data = dst;
+ ccw_update_attr(dst, src, attribute, vc);
+ src = dst;
+ }
+
+- if (ops->cursor_state.image.fg_color != fg ||
+- ops->cursor_state.image.bg_color != bg ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.fg_color = fg;
+- ops->cursor_state.image.bg_color = bg;
++ if (par->cursor_state.image.fg_color != fg ||
++ par->cursor_state.image.bg_color != bg ||
++ par->cursor_reset) {
++ par->cursor_state.image.fg_color = fg;
++ par->cursor_state.image.bg_color = bg;
+ cursor.set |= FB_CUR_SETCMAP;
+ }
+
+- if (ops->cursor_state.image.height != vc->vc_font.width ||
+- ops->cursor_state.image.width != vc->vc_font.height ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.height = vc->vc_font.width;
+- ops->cursor_state.image.width = vc->vc_font.height;
++ if (par->cursor_state.image.height != vc->vc_font.width ||
++ par->cursor_state.image.width != vc->vc_font.height ||
++ par->cursor_reset) {
++ par->cursor_state.image.height = vc->vc_font.width;
++ par->cursor_state.image.width = vc->vc_font.height;
+ cursor.set |= FB_CUR_SETSIZE;
+ }
+
+ dx = y * vc->vc_font.height;
+ dy = vyres - ((vc->state.x + 1) * vc->vc_font.width);
+
+- if (ops->cursor_state.image.dx != dx ||
+- ops->cursor_state.image.dy != dy ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.dx = dx;
+- ops->cursor_state.image.dy = dy;
++ if (par->cursor_state.image.dx != dx ||
++ par->cursor_state.image.dy != dy ||
++ par->cursor_reset) {
++ par->cursor_state.image.dx = dx;
++ par->cursor_state.image.dy = dy;
+ cursor.set |= FB_CUR_SETPOS;
+ }
+
+- if (ops->cursor_state.hot.x || ops->cursor_state.hot.y ||
+- ops->cursor_reset) {
+- ops->cursor_state.hot.x = cursor.hot.y = 0;
++ if (par->cursor_state.hot.x || par->cursor_state.hot.y ||
++ par->cursor_reset) {
++ par->cursor_state.hot.x = cursor.hot.y = 0;
+ cursor.set |= FB_CUR_SETHOT;
+ }
+
+ if (cursor.set & FB_CUR_SETSIZE ||
+- vc->vc_cursor_type != ops->p->cursor_shape ||
+- ops->cursor_state.mask == NULL ||
+- ops->cursor_reset) {
++ vc->vc_cursor_type != par->p->cursor_shape ||
++ par->cursor_state.mask == NULL ||
++ par->cursor_reset) {
+ char *tmp, *mask = kmalloc_array(w, vc->vc_font.width,
+ GFP_ATOMIC);
+ int cur_height, size, i = 0;
+@@ -309,13 +309,13 @@ static void ccw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ return;
+ }
+
+- kfree(ops->cursor_state.mask);
+- ops->cursor_state.mask = mask;
++ kfree(par->cursor_state.mask);
++ par->cursor_state.mask = mask;
+
+- ops->p->cursor_shape = vc->vc_cursor_type;
++ par->p->cursor_shape = vc->vc_cursor_type;
+ cursor.set |= FB_CUR_SETSHAPE;
+
+- switch (CUR_SIZE(ops->p->cursor_shape)) {
++ switch (CUR_SIZE(par->p->cursor_shape)) {
+ case CUR_NONE:
+ cur_height = 0;
+ break;
+@@ -348,19 +348,19 @@ static void ccw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ kfree(tmp);
+ }
+
+- ops->cursor_state.enable = enable && !use_sw;
++ par->cursor_state.enable = enable && !use_sw;
+
+ cursor.image.data = src;
+- cursor.image.fg_color = ops->cursor_state.image.fg_color;
+- cursor.image.bg_color = ops->cursor_state.image.bg_color;
+- cursor.image.dx = ops->cursor_state.image.dx;
+- cursor.image.dy = ops->cursor_state.image.dy;
+- cursor.image.height = ops->cursor_state.image.height;
+- cursor.image.width = ops->cursor_state.image.width;
+- cursor.hot.x = ops->cursor_state.hot.x;
+- cursor.hot.y = ops->cursor_state.hot.y;
+- cursor.mask = ops->cursor_state.mask;
+- cursor.enable = ops->cursor_state.enable;
++ cursor.image.fg_color = par->cursor_state.image.fg_color;
++ cursor.image.bg_color = par->cursor_state.image.bg_color;
++ cursor.image.dx = par->cursor_state.image.dx;
++ cursor.image.dy = par->cursor_state.image.dy;
++ cursor.image.height = par->cursor_state.image.height;
++ cursor.image.width = par->cursor_state.image.width;
++ cursor.hot.x = par->cursor_state.hot.x;
++ cursor.hot.y = par->cursor_state.hot.y;
++ cursor.mask = par->cursor_state.mask;
++ cursor.enable = par->cursor_state.enable;
+ cursor.image.depth = 1;
+ cursor.rop = ROP_XOR;
+
+@@ -370,32 +370,32 @@ static void ccw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ if (err)
+ soft_cursor(info, &cursor);
+
+- ops->cursor_reset = 0;
++ par->cursor_reset = 0;
+ }
+
+ static int ccw_update_start(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u32 yoffset;
+- u32 vyres = GETVYRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
+ int err;
+
+- yoffset = (vyres - info->var.yres) - ops->var.xoffset;
+- ops->var.xoffset = ops->var.yoffset;
+- ops->var.yoffset = yoffset;
+- err = fb_pan_display(info, &ops->var);
+- ops->var.xoffset = info->var.xoffset;
+- ops->var.yoffset = info->var.yoffset;
+- ops->var.vmode = info->var.vmode;
++ yoffset = (vyres - info->var.yres) - par->var.xoffset;
++ par->var.xoffset = par->var.yoffset;
++ par->var.yoffset = yoffset;
++ err = fb_pan_display(info, &par->var);
++ par->var.xoffset = info->var.xoffset;
++ par->var.yoffset = info->var.yoffset;
++ par->var.vmode = info->var.vmode;
+ return err;
+ }
+
+-void fbcon_rotate_ccw(struct fbcon_ops *ops)
++void fbcon_rotate_ccw(struct fbcon_par *par)
+ {
+- ops->bmove = ccw_bmove;
+- ops->clear = ccw_clear;
+- ops->putcs = ccw_putcs;
+- ops->clear_margins = ccw_clear_margins;
+- ops->cursor = ccw_cursor;
+- ops->update_start = ccw_update_start;
++ par->bmove = ccw_bmove;
++ par->clear = ccw_clear;
++ par->putcs = ccw_putcs;
++ par->clear_margins = ccw_clear_margins;
++ par->cursor = ccw_cursor;
++ par->update_start = ccw_update_start;
+ }
+diff --git a/drivers/video/fbdev/core/fbcon_cw.c b/drivers/video/fbdev/core/fbcon_cw.c
+index b9dac7940fb777..4bd22d5ee5f410 100644
+--- a/drivers/video/fbdev/core/fbcon_cw.c
++++ b/drivers/video/fbdev/core/fbcon_cw.c
+@@ -48,9 +48,9 @@ static void cw_update_attr(u8 *dst, u8 *src, int attribute,
+ static void cw_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ int sx, int dy, int dx, int height, int width)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fb_copyarea area;
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+ area.sx = vxres - ((sy + height) * vc->vc_font.height);
+ area.sy = sx * vc->vc_font.width;
+@@ -65,9 +65,9 @@ static void cw_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ static void cw_clear(struct vc_data *vc, struct fb_info *info, int sy,
+ int sx, int height, int width, int fg, int bg)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fb_fillrect region;
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+ region.color = bg;
+ region.dx = vxres - ((sy + height) * vc->vc_font.height);
+@@ -84,13 +84,13 @@ static inline void cw_putcs_aligned(struct vc_data *vc, struct fb_info *info,
+ u32 d_pitch, u32 s_pitch, u32 cellsize,
+ struct fb_image *image, u8 *buf, u8 *dst)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u16 charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ u32 idx = (vc->vc_font.height + 7) >> 3;
+ u8 *src;
+
+ while (cnt--) {
+- src = ops->fontbuffer + (scr_readw(s++) & charmask)*cellsize;
++ src = par->fontbuffer + (scr_readw(s++) & charmask) * cellsize;
+
+ if (attr) {
+ cw_update_attr(buf, src, attr, vc);
+@@ -115,7 +115,7 @@ static void cw_putcs(struct vc_data *vc, struct fb_info *info,
+ int fg, int bg)
+ {
+ struct fb_image image;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u32 width = (vc->vc_font.height + 7)/8;
+ u32 cellsize = width * vc->vc_font.width;
+ u32 maxcnt = info->pixmap.size/cellsize;
+@@ -124,9 +124,9 @@ static void cw_putcs(struct vc_data *vc, struct fb_info *info,
+ u32 cnt, pitch, size;
+ u32 attribute = get_attribute(info, scr_readw(s));
+ u8 *dst, *buf = NULL;
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+- if (!ops->fontbuffer)
++ if (!par->fontbuffer)
+ return;
+
+ image.fg_color = fg;
+@@ -204,28 +204,28 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ int fg, int bg)
+ {
+ struct fb_cursor cursor;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ unsigned short charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ int w = (vc->vc_font.height + 7) >> 3, c;
+- int y = real_y(ops->p, vc->state.y);
++ int y = real_y(par->p, vc->state.y);
+ int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
+ int err = 1, dx, dy;
+ char *src;
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+- if (!ops->fontbuffer)
++ if (!par->fontbuffer)
+ return;
+
+ cursor.set = 0;
+
+ c = scr_readw((u16 *) vc->vc_pos);
+ attribute = get_attribute(info, c);
+- src = ops->fontbuffer + ((c & charmask) * (w * vc->vc_font.width));
++ src = par->fontbuffer + ((c & charmask) * (w * vc->vc_font.width));
+
+- if (ops->cursor_state.image.data != src ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.data = src;
+- cursor.set |= FB_CUR_SETIMAGE;
++ if (par->cursor_state.image.data != src ||
++ par->cursor_reset) {
++ par->cursor_state.image.data = src;
++ cursor.set |= FB_CUR_SETIMAGE;
+ }
+
+ if (attribute) {
+@@ -234,49 +234,49 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ dst = kmalloc_array(w, vc->vc_font.width, GFP_ATOMIC);
+ if (!dst)
+ return;
+- kfree(ops->cursor_data);
+- ops->cursor_data = dst;
++ kfree(par->cursor_data);
++ par->cursor_data = dst;
+ cw_update_attr(dst, src, attribute, vc);
+ src = dst;
+ }
+
+- if (ops->cursor_state.image.fg_color != fg ||
+- ops->cursor_state.image.bg_color != bg ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.fg_color = fg;
+- ops->cursor_state.image.bg_color = bg;
++ if (par->cursor_state.image.fg_color != fg ||
++ par->cursor_state.image.bg_color != bg ||
++ par->cursor_reset) {
++ par->cursor_state.image.fg_color = fg;
++ par->cursor_state.image.bg_color = bg;
+ cursor.set |= FB_CUR_SETCMAP;
+ }
+
+- if (ops->cursor_state.image.height != vc->vc_font.width ||
+- ops->cursor_state.image.width != vc->vc_font.height ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.height = vc->vc_font.width;
+- ops->cursor_state.image.width = vc->vc_font.height;
++ if (par->cursor_state.image.height != vc->vc_font.width ||
++ par->cursor_state.image.width != vc->vc_font.height ||
++ par->cursor_reset) {
++ par->cursor_state.image.height = vc->vc_font.width;
++ par->cursor_state.image.width = vc->vc_font.height;
+ cursor.set |= FB_CUR_SETSIZE;
+ }
+
+ dx = vxres - ((y * vc->vc_font.height) + vc->vc_font.height);
+ dy = vc->state.x * vc->vc_font.width;
+
+- if (ops->cursor_state.image.dx != dx ||
+- ops->cursor_state.image.dy != dy ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.dx = dx;
+- ops->cursor_state.image.dy = dy;
++ if (par->cursor_state.image.dx != dx ||
++ par->cursor_state.image.dy != dy ||
++ par->cursor_reset) {
++ par->cursor_state.image.dx = dx;
++ par->cursor_state.image.dy = dy;
+ cursor.set |= FB_CUR_SETPOS;
+ }
+
+- if (ops->cursor_state.hot.x || ops->cursor_state.hot.y ||
+- ops->cursor_reset) {
+- ops->cursor_state.hot.x = cursor.hot.y = 0;
++ if (par->cursor_state.hot.x || par->cursor_state.hot.y ||
++ par->cursor_reset) {
++ par->cursor_state.hot.x = cursor.hot.y = 0;
+ cursor.set |= FB_CUR_SETHOT;
+ }
+
+ if (cursor.set & FB_CUR_SETSIZE ||
+- vc->vc_cursor_type != ops->p->cursor_shape ||
+- ops->cursor_state.mask == NULL ||
+- ops->cursor_reset) {
++ vc->vc_cursor_type != par->p->cursor_shape ||
++ par->cursor_state.mask == NULL ||
++ par->cursor_reset) {
+ char *tmp, *mask = kmalloc_array(w, vc->vc_font.width,
+ GFP_ATOMIC);
+ int cur_height, size, i = 0;
+@@ -292,13 +292,13 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ return;
+ }
+
+- kfree(ops->cursor_state.mask);
+- ops->cursor_state.mask = mask;
++ kfree(par->cursor_state.mask);
++ par->cursor_state.mask = mask;
+
+- ops->p->cursor_shape = vc->vc_cursor_type;
++ par->p->cursor_shape = vc->vc_cursor_type;
+ cursor.set |= FB_CUR_SETSHAPE;
+
+- switch (CUR_SIZE(ops->p->cursor_shape)) {
++ switch (CUR_SIZE(par->p->cursor_shape)) {
+ case CUR_NONE:
+ cur_height = 0;
+ break;
+@@ -331,19 +331,19 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ kfree(tmp);
+ }
+
+- ops->cursor_state.enable = enable && !use_sw;
++ par->cursor_state.enable = enable && !use_sw;
+
+ cursor.image.data = src;
+- cursor.image.fg_color = ops->cursor_state.image.fg_color;
+- cursor.image.bg_color = ops->cursor_state.image.bg_color;
+- cursor.image.dx = ops->cursor_state.image.dx;
+- cursor.image.dy = ops->cursor_state.image.dy;
+- cursor.image.height = ops->cursor_state.image.height;
+- cursor.image.width = ops->cursor_state.image.width;
+- cursor.hot.x = ops->cursor_state.hot.x;
+- cursor.hot.y = ops->cursor_state.hot.y;
+- cursor.mask = ops->cursor_state.mask;
+- cursor.enable = ops->cursor_state.enable;
++ cursor.image.fg_color = par->cursor_state.image.fg_color;
++ cursor.image.bg_color = par->cursor_state.image.bg_color;
++ cursor.image.dx = par->cursor_state.image.dx;
++ cursor.image.dy = par->cursor_state.image.dy;
++ cursor.image.height = par->cursor_state.image.height;
++ cursor.image.width = par->cursor_state.image.width;
++ cursor.hot.x = par->cursor_state.hot.x;
++ cursor.hot.y = par->cursor_state.hot.y;
++ cursor.mask = par->cursor_state.mask;
++ cursor.enable = par->cursor_state.enable;
+ cursor.image.depth = 1;
+ cursor.rop = ROP_XOR;
+
+@@ -353,32 +353,32 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ if (err)
+ soft_cursor(info, &cursor);
+
+- ops->cursor_reset = 0;
++ par->cursor_reset = 0;
+ }
+
+ static int cw_update_start(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
+- u32 vxres = GETVXRES(ops->p, info);
++ struct fbcon_par *par = info->fbcon_par;
++ u32 vxres = GETVXRES(par->p, info);
+ u32 xoffset;
+ int err;
+
+- xoffset = vxres - (info->var.xres + ops->var.yoffset);
+- ops->var.yoffset = ops->var.xoffset;
+- ops->var.xoffset = xoffset;
+- err = fb_pan_display(info, &ops->var);
+- ops->var.xoffset = info->var.xoffset;
+- ops->var.yoffset = info->var.yoffset;
+- ops->var.vmode = info->var.vmode;
++ xoffset = vxres - (info->var.xres + par->var.yoffset);
++ par->var.yoffset = par->var.xoffset;
++ par->var.xoffset = xoffset;
++ err = fb_pan_display(info, &par->var);
++ par->var.xoffset = info->var.xoffset;
++ par->var.yoffset = info->var.yoffset;
++ par->var.vmode = info->var.vmode;
+ return err;
+ }
+
+-void fbcon_rotate_cw(struct fbcon_ops *ops)
++void fbcon_rotate_cw(struct fbcon_par *par)
+ {
+- ops->bmove = cw_bmove;
+- ops->clear = cw_clear;
+- ops->putcs = cw_putcs;
+- ops->clear_margins = cw_clear_margins;
+- ops->cursor = cw_cursor;
+- ops->update_start = cw_update_start;
++ par->bmove = cw_bmove;
++ par->clear = cw_clear;
++ par->putcs = cw_putcs;
++ par->clear_margins = cw_clear_margins;
++ par->cursor = cw_cursor;
++ par->update_start = cw_update_start;
+ }
+diff --git a/drivers/video/fbdev/core/fbcon_rotate.c b/drivers/video/fbdev/core/fbcon_rotate.c
+index 4a06e71ae4434a..a3f507825eed8a 100644
+--- a/drivers/video/fbdev/core/fbcon_rotate.c
++++ b/drivers/video/fbdev/core/fbcon_rotate.c
+@@ -20,35 +20,35 @@
+
+ static int fbcon_rotate_font(struct fb_info *info, struct vc_data *vc)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int len, err = 0;
+ int s_cellsize, d_cellsize, i;
+ const u8 *src;
+ u8 *dst;
+
+- if (vc->vc_font.data == ops->fontdata &&
+- ops->p->con_rotate == ops->cur_rotate)
++ if (vc->vc_font.data == par->fontdata &&
++ par->p->con_rotate == par->cur_rotate)
+ goto finished;
+
+- src = ops->fontdata = vc->vc_font.data;
+- ops->cur_rotate = ops->p->con_rotate;
++ src = par->fontdata = vc->vc_font.data;
++ par->cur_rotate = par->p->con_rotate;
+ len = vc->vc_font.charcount;
+ s_cellsize = ((vc->vc_font.width + 7)/8) *
+ vc->vc_font.height;
+ d_cellsize = s_cellsize;
+
+- if (ops->rotate == FB_ROTATE_CW ||
+- ops->rotate == FB_ROTATE_CCW)
++ if (par->rotate == FB_ROTATE_CW ||
++ par->rotate == FB_ROTATE_CCW)
+ d_cellsize = ((vc->vc_font.height + 7)/8) *
+ vc->vc_font.width;
+
+ if (info->fbops->fb_sync)
+ info->fbops->fb_sync(info);
+
+- if (ops->fd_size < d_cellsize * len) {
+- kfree(ops->fontbuffer);
+- ops->fontbuffer = NULL;
+- ops->fd_size = 0;
++ if (par->fd_size < d_cellsize * len) {
++ kfree(par->fontbuffer);
++ par->fontbuffer = NULL;
++ par->fd_size = 0;
+
+ dst = kmalloc_array(len, d_cellsize, GFP_KERNEL);
+
+@@ -57,14 +57,14 @@ static int fbcon_rotate_font(struct fb_info *info, struct vc_data *vc)
+ goto finished;
+ }
+
+- ops->fd_size = d_cellsize * len;
+- ops->fontbuffer = dst;
++ par->fd_size = d_cellsize * len;
++ par->fontbuffer = dst;
+ }
+
+- dst = ops->fontbuffer;
+- memset(dst, 0, ops->fd_size);
++ dst = par->fontbuffer;
++ memset(dst, 0, par->fd_size);
+
+- switch (ops->rotate) {
++ switch (par->rotate) {
+ case FB_ROTATE_UD:
+ for (i = len; i--; ) {
+ rotate_ud(src, dst, vc->vc_font.width,
+@@ -96,19 +96,19 @@ finished:
+ return err;
+ }
+
+-void fbcon_set_rotate(struct fbcon_ops *ops)
++void fbcon_set_rotate(struct fbcon_par *par)
+ {
+- ops->rotate_font = fbcon_rotate_font;
++ par->rotate_font = fbcon_rotate_font;
+
+- switch(ops->rotate) {
++ switch (par->rotate) {
+ case FB_ROTATE_CW:
+- fbcon_rotate_cw(ops);
++ fbcon_rotate_cw(par);
+ break;
+ case FB_ROTATE_UD:
+- fbcon_rotate_ud(ops);
++ fbcon_rotate_ud(par);
+ break;
+ case FB_ROTATE_CCW:
+- fbcon_rotate_ccw(ops);
++ fbcon_rotate_ccw(par);
+ break;
+ }
+ }
+diff --git a/drivers/video/fbdev/core/fbcon_rotate.h b/drivers/video/fbdev/core/fbcon_rotate.h
+index 01cbe303b8a295..48305e1a07631f 100644
+--- a/drivers/video/fbdev/core/fbcon_rotate.h
++++ b/drivers/video/fbdev/core/fbcon_rotate.h
+@@ -90,7 +90,7 @@ static inline void rotate_ccw(const char *in, char *out, u32 width, u32 height)
+ }
+ }
+
+-extern void fbcon_rotate_cw(struct fbcon_ops *ops);
+-extern void fbcon_rotate_ud(struct fbcon_ops *ops);
+-extern void fbcon_rotate_ccw(struct fbcon_ops *ops);
++extern void fbcon_rotate_cw(struct fbcon_par *par);
++extern void fbcon_rotate_ud(struct fbcon_par *par);
++extern void fbcon_rotate_ccw(struct fbcon_par *par);
+ #endif
+diff --git a/drivers/video/fbdev/core/fbcon_ud.c b/drivers/video/fbdev/core/fbcon_ud.c
+index 0af7913a2abdcc..14b40e2bf323f9 100644
+--- a/drivers/video/fbdev/core/fbcon_ud.c
++++ b/drivers/video/fbdev/core/fbcon_ud.c
+@@ -48,10 +48,10 @@ static void ud_update_attr(u8 *dst, u8 *src, int attribute,
+ static void ud_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ int sx, int dy, int dx, int height, int width)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fb_copyarea area;
+- u32 vyres = GETVYRES(ops->p, info);
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+ area.sy = vyres - ((sy + height) * vc->vc_font.height);
+ area.sx = vxres - ((sx + width) * vc->vc_font.width);
+@@ -66,10 +66,10 @@ static void ud_bmove(struct vc_data *vc, struct fb_info *info, int sy,
+ static void ud_clear(struct vc_data *vc, struct fb_info *info, int sy,
+ int sx, int height, int width, int fg, int bg)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ struct fb_fillrect region;
+- u32 vyres = GETVYRES(ops->p, info);
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+ region.color = bg;
+ region.dy = vyres - ((sy + height) * vc->vc_font.height);
+@@ -86,13 +86,13 @@ static inline void ud_putcs_aligned(struct vc_data *vc, struct fb_info *info,
+ u32 d_pitch, u32 s_pitch, u32 cellsize,
+ struct fb_image *image, u8 *buf, u8 *dst)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u16 charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ u32 idx = vc->vc_font.width >> 3;
+ u8 *src;
+
+ while (cnt--) {
+- src = ops->fontbuffer + (scr_readw(s--) & charmask)*cellsize;
++ src = par->fontbuffer + (scr_readw(s--) & charmask) * cellsize;
+
+ if (attr) {
+ ud_update_attr(buf, src, attr, vc);
+@@ -119,7 +119,7 @@ static inline void ud_putcs_unaligned(struct vc_data *vc,
+ struct fb_image *image, u8 *buf,
+ u8 *dst)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u16 charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ u32 shift_low = 0, mod = vc->vc_font.width % 8;
+ u32 shift_high = 8;
+@@ -127,7 +127,7 @@ static inline void ud_putcs_unaligned(struct vc_data *vc,
+ u8 *src;
+
+ while (cnt--) {
+- src = ops->fontbuffer + (scr_readw(s--) & charmask)*cellsize;
++ src = par->fontbuffer + (scr_readw(s--) & charmask) * cellsize;
+
+ if (attr) {
+ ud_update_attr(buf, src, attr, vc);
+@@ -152,7 +152,7 @@ static void ud_putcs(struct vc_data *vc, struct fb_info *info,
+ int fg, int bg)
+ {
+ struct fb_image image;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ u32 width = (vc->vc_font.width + 7)/8;
+ u32 cellsize = width * vc->vc_font.height;
+ u32 maxcnt = info->pixmap.size/cellsize;
+@@ -161,10 +161,10 @@ static void ud_putcs(struct vc_data *vc, struct fb_info *info,
+ u32 mod = vc->vc_font.width % 8, cnt, pitch, size;
+ u32 attribute = get_attribute(info, scr_readw(s));
+ u8 *dst, *buf = NULL;
+- u32 vyres = GETVYRES(ops->p, info);
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+- if (!ops->fontbuffer)
++ if (!par->fontbuffer)
+ return;
+
+ image.fg_color = fg;
+@@ -251,29 +251,29 @@ static void ud_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ int fg, int bg)
+ {
+ struct fb_cursor cursor;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ unsigned short charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
+ int w = (vc->vc_font.width + 7) >> 3, c;
+- int y = real_y(ops->p, vc->state.y);
++ int y = real_y(par->p, vc->state.y);
+ int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
+ int err = 1, dx, dy;
+ char *src;
+- u32 vyres = GETVYRES(ops->p, info);
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+
+- if (!ops->fontbuffer)
++ if (!par->fontbuffer)
+ return;
+
+ cursor.set = 0;
+
+ c = scr_readw((u16 *) vc->vc_pos);
+ attribute = get_attribute(info, c);
+- src = ops->fontbuffer + ((c & charmask) * (w * vc->vc_font.height));
++ src = par->fontbuffer + ((c & charmask) * (w * vc->vc_font.height));
+
+- if (ops->cursor_state.image.data != src ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.data = src;
+- cursor.set |= FB_CUR_SETIMAGE;
++ if (par->cursor_state.image.data != src ||
++ par->cursor_reset) {
++ par->cursor_state.image.data = src;
++ cursor.set |= FB_CUR_SETIMAGE;
+ }
+
+ if (attribute) {
+@@ -282,49 +282,49 @@ static void ud_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ dst = kmalloc_array(w, vc->vc_font.height, GFP_ATOMIC);
+ if (!dst)
+ return;
+- kfree(ops->cursor_data);
+- ops->cursor_data = dst;
++ kfree(par->cursor_data);
++ par->cursor_data = dst;
+ ud_update_attr(dst, src, attribute, vc);
+ src = dst;
+ }
+
+- if (ops->cursor_state.image.fg_color != fg ||
+- ops->cursor_state.image.bg_color != bg ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.fg_color = fg;
+- ops->cursor_state.image.bg_color = bg;
++ if (par->cursor_state.image.fg_color != fg ||
++ par->cursor_state.image.bg_color != bg ||
++ par->cursor_reset) {
++ par->cursor_state.image.fg_color = fg;
++ par->cursor_state.image.bg_color = bg;
+ cursor.set |= FB_CUR_SETCMAP;
+ }
+
+- if (ops->cursor_state.image.height != vc->vc_font.height ||
+- ops->cursor_state.image.width != vc->vc_font.width ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.height = vc->vc_font.height;
+- ops->cursor_state.image.width = vc->vc_font.width;
++ if (par->cursor_state.image.height != vc->vc_font.height ||
++ par->cursor_state.image.width != vc->vc_font.width ||
++ par->cursor_reset) {
++ par->cursor_state.image.height = vc->vc_font.height;
++ par->cursor_state.image.width = vc->vc_font.width;
+ cursor.set |= FB_CUR_SETSIZE;
+ }
+
+ dy = vyres - ((y * vc->vc_font.height) + vc->vc_font.height);
+ dx = vxres - ((vc->state.x * vc->vc_font.width) + vc->vc_font.width);
+
+- if (ops->cursor_state.image.dx != dx ||
+- ops->cursor_state.image.dy != dy ||
+- ops->cursor_reset) {
+- ops->cursor_state.image.dx = dx;
+- ops->cursor_state.image.dy = dy;
++ if (par->cursor_state.image.dx != dx ||
++ par->cursor_state.image.dy != dy ||
++ par->cursor_reset) {
++ par->cursor_state.image.dx = dx;
++ par->cursor_state.image.dy = dy;
+ cursor.set |= FB_CUR_SETPOS;
+ }
+
+- if (ops->cursor_state.hot.x || ops->cursor_state.hot.y ||
+- ops->cursor_reset) {
+- ops->cursor_state.hot.x = cursor.hot.y = 0;
++ if (par->cursor_state.hot.x || par->cursor_state.hot.y ||
++ par->cursor_reset) {
++ par->cursor_state.hot.x = cursor.hot.y = 0;
+ cursor.set |= FB_CUR_SETHOT;
+ }
+
+ if (cursor.set & FB_CUR_SETSIZE ||
+- vc->vc_cursor_type != ops->p->cursor_shape ||
+- ops->cursor_state.mask == NULL ||
+- ops->cursor_reset) {
++ vc->vc_cursor_type != par->p->cursor_shape ||
++ par->cursor_state.mask == NULL ||
++ par->cursor_reset) {
+ char *mask = kmalloc_array(w, vc->vc_font.height, GFP_ATOMIC);
+ int cur_height, size, i = 0;
+ u8 msk = 0xff;
+@@ -332,13 +332,13 @@ static void ud_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ if (!mask)
+ return;
+
+- kfree(ops->cursor_state.mask);
+- ops->cursor_state.mask = mask;
++ kfree(par->cursor_state.mask);
++ par->cursor_state.mask = mask;
+
+- ops->p->cursor_shape = vc->vc_cursor_type;
++ par->p->cursor_shape = vc->vc_cursor_type;
+ cursor.set |= FB_CUR_SETSHAPE;
+
+- switch (CUR_SIZE(ops->p->cursor_shape)) {
++ switch (CUR_SIZE(par->p->cursor_shape)) {
+ case CUR_NONE:
+ cur_height = 0;
+ break;
+@@ -371,19 +371,19 @@ static void ud_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ mask[i++] = ~msk;
+ }
+
+- ops->cursor_state.enable = enable && !use_sw;
++ par->cursor_state.enable = enable && !use_sw;
+
+ cursor.image.data = src;
+- cursor.image.fg_color = ops->cursor_state.image.fg_color;
+- cursor.image.bg_color = ops->cursor_state.image.bg_color;
+- cursor.image.dx = ops->cursor_state.image.dx;
+- cursor.image.dy = ops->cursor_state.image.dy;
+- cursor.image.height = ops->cursor_state.image.height;
+- cursor.image.width = ops->cursor_state.image.width;
+- cursor.hot.x = ops->cursor_state.hot.x;
+- cursor.hot.y = ops->cursor_state.hot.y;
+- cursor.mask = ops->cursor_state.mask;
+- cursor.enable = ops->cursor_state.enable;
++ cursor.image.fg_color = par->cursor_state.image.fg_color;
++ cursor.image.bg_color = par->cursor_state.image.bg_color;
++ cursor.image.dx = par->cursor_state.image.dx;
++ cursor.image.dy = par->cursor_state.image.dy;
++ cursor.image.height = par->cursor_state.image.height;
++ cursor.image.width = par->cursor_state.image.width;
++ cursor.hot.x = par->cursor_state.hot.x;
++ cursor.hot.y = par->cursor_state.hot.y;
++ cursor.mask = par->cursor_state.mask;
++ cursor.enable = par->cursor_state.enable;
+ cursor.image.depth = 1;
+ cursor.rop = ROP_XOR;
+
+@@ -393,36 +393,36 @@ static void ud_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+ if (err)
+ soft_cursor(info, &cursor);
+
+- ops->cursor_reset = 0;
++ par->cursor_reset = 0;
+ }
+
+ static int ud_update_start(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int xoffset, yoffset;
+- u32 vyres = GETVYRES(ops->p, info);
+- u32 vxres = GETVXRES(ops->p, info);
++ u32 vyres = GETVYRES(par->p, info);
++ u32 vxres = GETVXRES(par->p, info);
+ int err;
+
+- xoffset = vxres - info->var.xres - ops->var.xoffset;
+- yoffset = vyres - info->var.yres - ops->var.yoffset;
++ xoffset = vxres - info->var.xres - par->var.xoffset;
++ yoffset = vyres - info->var.yres - par->var.yoffset;
+ if (yoffset < 0)
+ yoffset += vyres;
+- ops->var.xoffset = xoffset;
+- ops->var.yoffset = yoffset;
+- err = fb_pan_display(info, &ops->var);
+- ops->var.xoffset = info->var.xoffset;
+- ops->var.yoffset = info->var.yoffset;
+- ops->var.vmode = info->var.vmode;
++ par->var.xoffset = xoffset;
++ par->var.yoffset = yoffset;
++ err = fb_pan_display(info, &par->var);
++ par->var.xoffset = info->var.xoffset;
++ par->var.yoffset = info->var.yoffset;
++ par->var.vmode = info->var.vmode;
+ return err;
+ }
+
+-void fbcon_rotate_ud(struct fbcon_ops *ops)
++void fbcon_rotate_ud(struct fbcon_par *par)
+ {
+- ops->bmove = ud_bmove;
+- ops->clear = ud_clear;
+- ops->putcs = ud_putcs;
+- ops->clear_margins = ud_clear_margins;
+- ops->cursor = ud_cursor;
+- ops->update_start = ud_update_start;
++ par->bmove = ud_bmove;
++ par->clear = ud_clear;
++ par->putcs = ud_putcs;
++ par->clear_margins = ud_clear_margins;
++ par->cursor = ud_cursor;
++ par->update_start = ud_update_start;
+ }
+diff --git a/drivers/video/fbdev/core/softcursor.c b/drivers/video/fbdev/core/softcursor.c
+index 29e5b21cf373e5..900788c059153f 100644
+--- a/drivers/video/fbdev/core/softcursor.c
++++ b/drivers/video/fbdev/core/softcursor.c
+@@ -21,7 +21,7 @@
+
+ int soft_cursor(struct fb_info *info, struct fb_cursor *cursor)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ unsigned int scan_align = info->pixmap.scan_align - 1;
+ unsigned int buf_align = info->pixmap.buf_align - 1;
+ unsigned int i, size, dsize, s_pitch, d_pitch;
+@@ -34,19 +34,19 @@ int soft_cursor(struct fb_info *info, struct fb_cursor *cursor)
+ s_pitch = (cursor->image.width + 7) >> 3;
+ dsize = s_pitch * cursor->image.height;
+
+- if (dsize + sizeof(struct fb_image) != ops->cursor_size) {
+- kfree(ops->cursor_src);
+- ops->cursor_size = dsize + sizeof(struct fb_image);
++ if (dsize + sizeof(struct fb_image) != par->cursor_size) {
++ kfree(par->cursor_src);
++ par->cursor_size = dsize + sizeof(struct fb_image);
+
+- ops->cursor_src = kmalloc(ops->cursor_size, GFP_ATOMIC);
+- if (!ops->cursor_src) {
+- ops->cursor_size = 0;
++ par->cursor_src = kmalloc(par->cursor_size, GFP_ATOMIC);
++ if (!par->cursor_src) {
++ par->cursor_size = 0;
+ return -ENOMEM;
+ }
+ }
+
+- src = ops->cursor_src + sizeof(struct fb_image);
+- image = (struct fb_image *)ops->cursor_src;
++ src = par->cursor_src + sizeof(struct fb_image);
++ image = (struct fb_image *)par->cursor_src;
+ *image = cursor->image;
+ d_pitch = (s_pitch + scan_align) & ~scan_align;
+
+diff --git a/drivers/video/fbdev/core/tileblit.c b/drivers/video/fbdev/core/tileblit.c
+index d342b90c42b7fe..4428f2bcd3f8cd 100644
+--- a/drivers/video/fbdev/core/tileblit.c
++++ b/drivers/video/fbdev/core/tileblit.c
+@@ -151,34 +151,34 @@ static void tile_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
+
+ static int tile_update_start(struct fb_info *info)
+ {
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+ int err;
+
+- err = fb_pan_display(info, &ops->var);
+- ops->var.xoffset = info->var.xoffset;
+- ops->var.yoffset = info->var.yoffset;
+- ops->var.vmode = info->var.vmode;
++ err = fb_pan_display(info, &par->var);
++ par->var.xoffset = info->var.xoffset;
++ par->var.yoffset = info->var.yoffset;
++ par->var.vmode = info->var.vmode;
+ return err;
+ }
+
+ void fbcon_set_tileops(struct vc_data *vc, struct fb_info *info)
+ {
+ struct fb_tilemap map;
+- struct fbcon_ops *ops = info->fbcon_par;
++ struct fbcon_par *par = info->fbcon_par;
+
+- ops->bmove = tile_bmove;
+- ops->clear = tile_clear;
+- ops->putcs = tile_putcs;
+- ops->clear_margins = tile_clear_margins;
+- ops->cursor = tile_cursor;
+- ops->update_start = tile_update_start;
++ par->bmove = tile_bmove;
++ par->clear = tile_clear;
++ par->putcs = tile_putcs;
++ par->clear_margins = tile_clear_margins;
++ par->cursor = tile_cursor;
++ par->update_start = tile_update_start;
+
+- if (ops->p) {
++ if (par->p) {
+ map.width = vc->vc_font.width;
+ map.height = vc->vc_font.height;
+ map.depth = 1;
+ map.length = vc->vc_font.charcount;
+- map.data = ops->p->fontdata;
++ map.data = par->p->fontdata;
+ info->tileops->fb_settile(info, &map);
+ }
+ }
+diff --git a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c
+index 4230b817a80bd8..d66291def0f408 100644
+--- a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c
++++ b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c
+@@ -82,8 +82,8 @@ static int mmio_guard_ioremap_hook(phys_addr_t phys, size_t size,
+ if (protval != PROT_DEVICE_nGnRE && protval != PROT_DEVICE_nGnRnE)
+ return 0;
+
++ end = PAGE_ALIGN(phys + size);
+ phys = PAGE_ALIGN_DOWN(phys);
+- end = phys + PAGE_ALIGN(size);
+
+ while (phys < end) {
+ const int func_id = ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_FUNC_ID;
+diff --git a/drivers/watchdog/watchdog_pretimeout.c b/drivers/watchdog/watchdog_pretimeout.c
+index e5295c990fa1b8..787b6227b8ccc3 100644
+--- a/drivers/watchdog/watchdog_pretimeout.c
++++ b/drivers/watchdog/watchdog_pretimeout.c
+@@ -165,6 +165,8 @@ void watchdog_unregister_governor(struct watchdog_governor *gov)
+ }
+
+ spin_lock_irq(&pretimeout_lock);
++ if (default_gov == gov)
++ default_gov = NULL;
+ list_for_each_entry(p, &pretimeout_list, entry)
+ if (p->wdd->gov == gov)
+ p->wdd->gov = default_gov;
+diff --git a/fs/afs/addr_list.c b/fs/afs/addr_list.c
+index 6d42f85c6be54c..e941da5b6dd92d 100644
+--- a/fs/afs/addr_list.c
++++ b/fs/afs/addr_list.c
+@@ -362,3 +362,53 @@ int afs_merge_fs_addr6(struct afs_net *net, struct afs_addr_list *alist,
+ alist->nr_addrs++;
+ return 0;
+ }
++
++/*
++ * Set the app data on the rxrpc peers an address list points to
++ */
++void afs_set_peer_appdata(struct afs_server *server,
++ struct afs_addr_list *old_alist,
++ struct afs_addr_list *new_alist)
++{
++ unsigned long data = (unsigned long)server;
++ int n = 0, o = 0;
++
++ if (!old_alist) {
++ /* New server. Just set all. */
++ for (; n < new_alist->nr_addrs; n++)
++ rxrpc_kernel_set_peer_data(new_alist->addrs[n].peer, data);
++ return;
++ }
++ if (!new_alist) {
++ /* Dead server. Just remove all. */
++ for (; o < old_alist->nr_addrs; o++)
++ rxrpc_kernel_set_peer_data(old_alist->addrs[o].peer, 0);
++ return;
++ }
++
++ /* Walk through the two lists simultaneously, setting new peers and
++ * clearing old ones. The two lists are ordered by pointer to peer
++ * record.
++ */
++ while (n < new_alist->nr_addrs && o < old_alist->nr_addrs) {
++ struct rxrpc_peer *pn = new_alist->addrs[n].peer;
++ struct rxrpc_peer *po = old_alist->addrs[o].peer;
++
++ if (pn == po)
++ continue;
++ if (pn < po) {
++ rxrpc_kernel_set_peer_data(pn, data);
++ n++;
++ } else {
++ rxrpc_kernel_set_peer_data(po, 0);
++ o++;
++ }
++ }
++
++ if (n < new_alist->nr_addrs)
++ for (; n < new_alist->nr_addrs; n++)
++ rxrpc_kernel_set_peer_data(new_alist->addrs[n].peer, data);
++ if (o < old_alist->nr_addrs)
++ for (; o < old_alist->nr_addrs; o++)
++ rxrpc_kernel_set_peer_data(old_alist->addrs[o].peer, 0);
++}
+diff --git a/fs/afs/cell.c b/fs/afs/cell.c
+index ce74438b9fb821..e85e50716459eb 100644
+--- a/fs/afs/cell.c
++++ b/fs/afs/cell.c
+@@ -20,8 +20,9 @@ static unsigned __read_mostly afs_cell_min_ttl = 10 * 60;
+ static unsigned __read_mostly afs_cell_max_ttl = 24 * 60 * 60;
+ static atomic_t cell_debug_id;
+
+-static void afs_queue_cell_manager(struct afs_net *);
+-static void afs_manage_cell_work(struct work_struct *);
++static void afs_cell_timer(struct timer_list *timer);
++static void afs_destroy_cell_work(struct work_struct *work);
++static void afs_manage_cell_work(struct work_struct *work);
+
+ static void afs_dec_cells_outstanding(struct afs_net *net)
+ {
+@@ -29,19 +30,11 @@ static void afs_dec_cells_outstanding(struct afs_net *net)
+ wake_up_var(&net->cells_outstanding);
+ }
+
+-/*
+- * Set the cell timer to fire after a given delay, assuming it's not already
+- * set for an earlier time.
+- */
+-static void afs_set_cell_timer(struct afs_net *net, time64_t delay)
++static void afs_set_cell_state(struct afs_cell *cell, enum afs_cell_state state)
+ {
+- if (net->live) {
+- atomic_inc(&net->cells_outstanding);
+- if (timer_reduce(&net->cells_timer, jiffies + delay * HZ))
+- afs_dec_cells_outstanding(net);
+- } else {
+- afs_queue_cell_manager(net);
+- }
++ smp_store_release(&cell->state, state); /* Commit cell changes before state */
++ smp_wmb(); /* Set cell state before task state */
++ wake_up_var(&cell->state);
+ }
+
+ /*
+@@ -116,7 +109,7 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net,
+ const char *name, unsigned int namelen,
+ const char *addresses)
+ {
+- struct afs_vlserver_list *vllist;
++ struct afs_vlserver_list *vllist = NULL;
+ struct afs_cell *cell;
+ int i, ret;
+
+@@ -147,7 +140,9 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net,
+ return ERR_PTR(-ENOMEM);
+ }
+
+- cell->name = kmalloc(1 + namelen + 1, GFP_KERNEL);
++ /* Allocate the cell name and the key name in one go. */
++ cell->name = kmalloc(1 + namelen + 1 +
++ 4 + namelen + 1, GFP_KERNEL);
+ if (!cell->name) {
+ kfree(cell);
+ return ERR_PTR(-ENOMEM);
+@@ -158,18 +153,24 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net,
+ cell->name_len = namelen;
+ for (i = 0; i < namelen; i++)
+ cell->name[i] = tolower(name[i]);
+- cell->name[i] = 0;
++ cell->name[i++] = 0;
++
++ cell->key_desc = cell->name + i;
++ memcpy(cell->key_desc, "afs@", 4);
++ memcpy(cell->key_desc + 4, cell->name, cell->name_len + 1);
+
+ cell->net = net;
+ refcount_set(&cell->ref, 1);
+ atomic_set(&cell->active, 0);
++ INIT_WORK(&cell->destroyer, afs_destroy_cell_work);
+ INIT_WORK(&cell->manager, afs_manage_cell_work);
++ timer_setup(&cell->management_timer, afs_cell_timer, 0);
+ init_rwsem(&cell->vs_lock);
+ cell->volumes = RB_ROOT;
+ INIT_HLIST_HEAD(&cell->proc_volumes);
+ seqlock_init(&cell->volume_lock);
+ cell->fs_servers = RB_ROOT;
+- seqlock_init(&cell->fs_lock);
++ init_rwsem(&cell->fs_lock);
+ rwlock_init(&cell->vl_servers_lock);
+ cell->flags = (1 << AFS_CELL_FL_CHECK_ALIAS);
+
+@@ -182,6 +183,7 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net,
+ VL_SERVICE, AFS_VL_PORT);
+ if (IS_ERR(vllist)) {
+ ret = PTR_ERR(vllist);
++ vllist = NULL;
+ goto parse_failed;
+ }
+
+@@ -203,8 +205,10 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net,
+ cell->dns_source = vllist->source;
+ cell->dns_status = vllist->status;
+ smp_store_release(&cell->dns_lookup_count, 1); /* vs source/status */
++ down_write(&net->cells_lock);
+ ret = idr_alloc_cyclic(&net->cells_dyn_ino, cell,
+ 2, INT_MAX / 2, GFP_KERNEL);
++ up_write(&net->cells_lock);
+ if (ret < 0)
+ goto error;
+ atomic_inc(&net->cells_outstanding);
+@@ -220,6 +224,7 @@ parse_failed:
+ if (ret == -EINVAL)
+ printk(KERN_ERR "kAFS: bad VL server IP address\n");
+ error:
++ afs_put_vlserverlist(cell->net, vllist);
+ kfree(cell->name - 1);
+ kfree(cell);
+ _leave(" = %d", ret);
+@@ -232,7 +237,8 @@ error:
+ * @name: The name of the cell.
+ * @namesz: The strlen of the cell name.
+ * @vllist: A colon/comma separated list of numeric IP addresses or NULL.
+- * @excl: T if an error should be given if the cell name already exists.
++ * @reason: The reason we're doing the lookup
++ * @trace: The reason to be logged if the lookup is successful.
+ *
+ * Look up a cell record by name and query the DNS for VL server addresses if
+ * needed. Note that that actual DNS query is punted off to the manager thread
+@@ -241,19 +247,27 @@ error:
+ */
+ struct afs_cell *afs_lookup_cell(struct afs_net *net,
+ const char *name, unsigned int namesz,
+- const char *vllist, bool excl)
++ const char *vllist,
++ enum afs_lookup_cell_for reason,
++ enum afs_cell_trace trace)
+ {
+ struct afs_cell *cell, *candidate, *cursor;
+ struct rb_node *parent, **pp;
+ enum afs_cell_state state;
+ int ret, n;
+
+- _enter("%s,%s", name, vllist);
++ _enter("%s,%s,%u", name, vllist, reason);
+
+- if (!excl) {
+- cell = afs_find_cell(net, name, namesz, afs_cell_trace_use_lookup);
+- if (!IS_ERR(cell))
++ if (reason != AFS_LOOKUP_CELL_PRELOAD) {
++ cell = afs_find_cell(net, name, namesz, trace);
++ if (!IS_ERR(cell)) {
++ if (reason == AFS_LOOKUP_CELL_DYNROOT)
++ goto no_wait;
++ if (cell->state == AFS_CELL_SETTING_UP ||
++ cell->state == AFS_CELL_UNLOOKED)
++ goto lookup_cell;
+ goto wait_for_cell;
++ }
+ }
+
+ /* Assume we're probably going to create a cell and preallocate and
+@@ -294,29 +308,74 @@ struct afs_cell *afs_lookup_cell(struct afs_net *net,
+
+ cell = candidate;
+ candidate = NULL;
+- atomic_set(&cell->active, 2);
+- trace_afs_cell(cell->debug_id, refcount_read(&cell->ref), 2, afs_cell_trace_insert);
++ afs_use_cell(cell, trace);
+ rb_link_node_rcu(&cell->net_node, parent, pp);
+ rb_insert_color(&cell->net_node, &net->cells);
+ up_write(&net->cells_lock);
+
+- afs_queue_cell(cell, afs_cell_trace_get_queue_new);
++lookup_cell:
++ if (reason != AFS_LOOKUP_CELL_PRELOAD &&
++ reason != AFS_LOOKUP_CELL_ROOTCELL) {
++ set_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags);
++ afs_queue_cell(cell, afs_cell_trace_queue_new);
++ }
+
+ wait_for_cell:
+- trace_afs_cell(cell->debug_id, refcount_read(&cell->ref), atomic_read(&cell->active),
+- afs_cell_trace_wait);
+- _debug("wait_for_cell");
+- wait_var_event(&cell->state,
+- ({
+- state = smp_load_acquire(&cell->state); /* vs error */
+- state == AFS_CELL_ACTIVE || state == AFS_CELL_REMOVED;
+- }));
++ state = smp_load_acquire(&cell->state); /* vs error */
++ switch (state) {
++ case AFS_CELL_ACTIVE:
++ case AFS_CELL_DEAD:
++ break;
++ case AFS_CELL_UNLOOKED:
++ default:
++ if (reason == AFS_LOOKUP_CELL_PRELOAD ||
++ reason == AFS_LOOKUP_CELL_ROOTCELL)
++ break;
++ _debug("wait_for_cell");
++ afs_see_cell(cell, afs_cell_trace_wait);
++ wait_var_event(&cell->state,
++ ({
++ state = smp_load_acquire(&cell->state); /* vs error */
++ state == AFS_CELL_ACTIVE || state == AFS_CELL_DEAD;
++ }));
++ _debug("waited_for_cell %d %d", cell->state, cell->error);
++ }
+
++no_wait:
+ /* Check the state obtained from the wait check. */
+- if (state == AFS_CELL_REMOVED) {
++ state = smp_load_acquire(&cell->state); /* vs error */
++ if (state == AFS_CELL_DEAD) {
+ ret = cell->error;
+ goto error;
+ }
++ if (state == AFS_CELL_ACTIVE) {
++ switch (cell->dns_status) {
++ case DNS_LOOKUP_NOT_DONE:
++ if (cell->dns_source == DNS_RECORD_FROM_CONFIG) {
++ ret = 0;
++ break;
++ }
++ fallthrough;
++ default:
++ ret = -EIO;
++ goto error;
++ case DNS_LOOKUP_GOOD:
++ case DNS_LOOKUP_GOOD_WITH_BAD:
++ ret = 0;
++ break;
++ case DNS_LOOKUP_GOT_NOT_FOUND:
++ ret = -ENOENT;
++ goto error;
++ case DNS_LOOKUP_BAD:
++ ret = -EREMOTEIO;
++ goto error;
++ case DNS_LOOKUP_GOT_LOCAL_FAILURE:
++ case DNS_LOOKUP_GOT_TEMP_FAILURE:
++ case DNS_LOOKUP_GOT_NS_FAILURE:
++ ret = -EDESTADDRREQ;
++ goto error;
++ }
++ }
+
+ _leave(" = %p [cell]", cell);
+ return cell;
+@@ -324,10 +383,10 @@ wait_for_cell:
+ cell_already_exists:
+ _debug("cell exists");
+ cell = cursor;
+- if (excl) {
++ if (reason == AFS_LOOKUP_CELL_PRELOAD) {
+ ret = -EEXIST;
+ } else {
+- afs_use_cell(cursor, afs_cell_trace_use_lookup);
++ afs_use_cell(cursor, trace);
+ ret = 0;
+ }
+ up_write(&net->cells_lock);
+@@ -337,7 +396,7 @@ cell_already_exists:
+ goto wait_for_cell;
+ goto error_noput;
+ error:
+- afs_unuse_cell(net, cell, afs_cell_trace_unuse_lookup);
++ afs_unuse_cell(cell, afs_cell_trace_unuse_lookup_error);
+ error_noput:
+ _leave(" = %d [error]", ret);
+ return ERR_PTR(ret);
+@@ -382,8 +441,10 @@ int afs_cell_init(struct afs_net *net, const char *rootcell)
+ if (cp && cp < rootcell + len)
+ return -EINVAL;
+
+- /* allocate a cell record for the root cell */
+- new_root = afs_lookup_cell(net, rootcell, len, vllist, false);
++ /* allocate a cell record for the root/workstation cell */
++ new_root = afs_lookup_cell(net, rootcell, len, vllist,
++ AFS_LOOKUP_CELL_ROOTCELL,
++ afs_cell_trace_use_lookup_ws);
+ if (IS_ERR(new_root)) {
+ _leave(" = %ld", PTR_ERR(new_root));
+ return PTR_ERR(new_root);
+@@ -394,12 +455,11 @@ int afs_cell_init(struct afs_net *net, const char *rootcell)
+
+ /* install the new cell */
+ down_write(&net->cells_lock);
+- afs_see_cell(new_root, afs_cell_trace_see_ws);
+ old_root = rcu_replace_pointer(net->ws_cell, new_root,
+ lockdep_is_held(&net->cells_lock));
+ up_write(&net->cells_lock);
+
+- afs_unuse_cell(net, old_root, afs_cell_trace_unuse_ws);
++ afs_unuse_cell(old_root, afs_cell_trace_unuse_ws);
+ _leave(" = 0");
+ return 0;
+ }
+@@ -519,9 +579,8 @@ static void afs_cell_destroy(struct rcu_head *rcu)
+ trace_afs_cell(cell->debug_id, r, atomic_read(&cell->active), afs_cell_trace_free);
+
+ afs_put_vlserverlist(net, rcu_access_pointer(cell->vl_servers));
+- afs_unuse_cell(net, cell->alias_of, afs_cell_trace_unuse_alias);
++ afs_unuse_cell(cell->alias_of, afs_cell_trace_unuse_alias);
+ key_put(cell->anonymous_key);
+- idr_remove(&net->cells_dyn_ino, cell->dynroot_ino);
+ kfree(cell->name - 1);
+ kfree(cell);
+
+@@ -529,30 +588,19 @@ static void afs_cell_destroy(struct rcu_head *rcu)
+ _leave(" [destroyed]");
+ }
+
+-/*
+- * Queue the cell manager.
+- */
+-static void afs_queue_cell_manager(struct afs_net *net)
++static void afs_destroy_cell_work(struct work_struct *work)
+ {
+- int outstanding = atomic_inc_return(&net->cells_outstanding);
++ struct afs_cell *cell = container_of(work, struct afs_cell, destroyer);
+
+- _enter("%d", outstanding);
++ afs_see_cell(cell, afs_cell_trace_destroy);
++ timer_delete_sync(&cell->management_timer);
++ cancel_work_sync(&cell->manager);
+
+- if (!queue_work(afs_wq, &net->cells_manager))
+- afs_dec_cells_outstanding(net);
+-}
+-
+-/*
+- * Cell management timer. We have an increment on cells_outstanding that we
+- * need to pass along to the work item.
+- */
+-void afs_cells_timer(struct timer_list *timer)
+-{
+- struct afs_net *net = container_of(timer, struct afs_net, cells_timer);
++ down_write(&cell->net->cells_lock);
++ idr_remove(&cell->net->cells_dyn_ino, cell->dynroot_ino);
++ up_write(&cell->net->cells_lock);
+
+- _enter("");
+- if (!queue_work(afs_wq, &net->cells_manager))
+- afs_dec_cells_outstanding(net);
++ call_rcu(&cell->rcu, afs_cell_destroy);
+ }
+
+ /*
+@@ -584,7 +632,7 @@ void afs_put_cell(struct afs_cell *cell, enum afs_cell_trace reason)
+ if (zero) {
+ a = atomic_read(&cell->active);
+ WARN(a != 0, "Cell active count %u > 0\n", a);
+- call_rcu(&cell->rcu, afs_cell_destroy);
++ WARN_ON(!queue_work(afs_wq, &cell->destroyer));
+ }
+ }
+ }
+@@ -596,10 +644,9 @@ struct afs_cell *afs_use_cell(struct afs_cell *cell, enum afs_cell_trace reason)
+ {
+ int r, a;
+
+- r = refcount_read(&cell->ref);
+- WARN_ON(r == 0);
++ __refcount_inc(&cell->ref, &r);
+ a = atomic_inc_return(&cell->active);
+- trace_afs_cell(cell->debug_id, r, a, reason);
++ trace_afs_cell(cell->debug_id, r + 1, a, reason);
+ return cell;
+ }
+
+@@ -607,10 +654,11 @@ struct afs_cell *afs_use_cell(struct afs_cell *cell, enum afs_cell_trace reason)
+ * Record a cell becoming less active. When the active counter reaches 1, it
+ * is scheduled for destruction, but may get reactivated.
+ */
+-void afs_unuse_cell(struct afs_net *net, struct afs_cell *cell, enum afs_cell_trace reason)
++void afs_unuse_cell(struct afs_cell *cell, enum afs_cell_trace reason)
+ {
+ unsigned int debug_id;
+ time64_t now, expire_delay;
++ bool zero;
+ int r, a;
+
+ if (!cell)
+@@ -625,13 +673,15 @@ void afs_unuse_cell(struct afs_net *net, struct afs_cell *cell, enum afs_cell_tr
+ expire_delay = afs_cell_gc_delay;
+
+ debug_id = cell->debug_id;
+- r = refcount_read(&cell->ref);
+ a = atomic_dec_return(&cell->active);
+- trace_afs_cell(debug_id, r, a, reason);
+- WARN_ON(a == 0);
+- if (a == 1)
++ if (!a)
+ /* 'cell' may now be garbage collected. */
+- afs_set_cell_timer(net, expire_delay);
++ afs_set_cell_timer(cell, expire_delay);
++
++ zero = __refcount_dec_and_test(&cell->ref, &r);
++ trace_afs_cell(debug_id, r - 1, a, reason);
++ if (zero)
++ WARN_ON(!queue_work(afs_wq, &cell->destroyer));
+ }
+
+ /*
+@@ -651,36 +701,27 @@ void afs_see_cell(struct afs_cell *cell, enum afs_cell_trace reason)
+ */
+ void afs_queue_cell(struct afs_cell *cell, enum afs_cell_trace reason)
+ {
+- afs_get_cell(cell, reason);
+- if (!queue_work(afs_wq, &cell->manager))
+- afs_put_cell(cell, afs_cell_trace_put_queue_fail);
++ queue_work(afs_wq, &cell->manager);
+ }
+
+ /*
+- * Allocate a key to use as a placeholder for anonymous user security.
++ * Cell-specific management timer.
+ */
+-static int afs_alloc_anon_key(struct afs_cell *cell)
++static void afs_cell_timer(struct timer_list *timer)
+ {
+- struct key *key;
+- char keyname[4 + AFS_MAXCELLNAME + 1], *cp, *dp;
+-
+- /* Create a key to represent an anonymous user. */
+- memcpy(keyname, "afs@", 4);
+- dp = keyname + 4;
+- cp = cell->name;
+- do {
+- *dp++ = tolower(*cp);
+- } while (*cp++);
++ struct afs_cell *cell = container_of(timer, struct afs_cell, management_timer);
+
+- key = rxrpc_get_null_key(keyname);
+- if (IS_ERR(key))
+- return PTR_ERR(key);
+-
+- cell->anonymous_key = key;
++ afs_see_cell(cell, afs_cell_trace_see_mgmt_timer);
++ if (refcount_read(&cell->ref) > 0 && cell->net->live)
++ queue_work(afs_wq, &cell->manager);
++}
+
+- _debug("anon key %p{%x}",
+- cell->anonymous_key, key_serial(cell->anonymous_key));
+- return 0;
++/*
++ * Set/reduce the cell timer.
++ */
++void afs_set_cell_timer(struct afs_cell *cell, unsigned int delay_secs)
++{
++ timer_reduce(&cell->management_timer, jiffies + delay_secs * HZ);
+ }
+
+ /*
+@@ -692,12 +733,6 @@ static int afs_activate_cell(struct afs_net *net, struct afs_cell *cell)
+ struct afs_cell *pcell;
+ int ret;
+
+- if (!cell->anonymous_key) {
+- ret = afs_alloc_anon_key(cell);
+- if (ret < 0)
+- return ret;
+- }
+-
+ ret = afs_proc_cell_setup(cell);
+ if (ret < 0)
+ return ret;
+@@ -736,211 +771,128 @@ static void afs_deactivate_cell(struct afs_net *net, struct afs_cell *cell)
+ _leave("");
+ }
+
++static bool afs_has_cell_expired(struct afs_cell *cell, time64_t *_next_manage)
++{
++ const struct afs_vlserver_list *vllist;
++ time64_t expire_at = cell->last_inactive;
++ time64_t now = ktime_get_real_seconds();
++
++ if (atomic_read(&cell->active))
++ return false;
++ if (!cell->net->live)
++ return true;
++
++ vllist = rcu_dereference_protected(cell->vl_servers, true);
++ if (vllist && vllist->nr_servers > 0)
++ expire_at += afs_cell_gc_delay;
++
++ if (expire_at <= now)
++ return true;
++ if (expire_at < *_next_manage)
++ *_next_manage = expire_at;
++ return false;
++}
++
+ /*
+ * Manage a cell record, initialising and destroying it, maintaining its DNS
+ * records.
+ */
+-static void afs_manage_cell(struct afs_cell *cell)
++static bool afs_manage_cell(struct afs_cell *cell)
+ {
+ struct afs_net *net = cell->net;
+- int ret, active;
++ time64_t next_manage = TIME64_MAX;
++ int ret;
+
+ _enter("%s", cell->name);
+
+-again:
+ _debug("state %u", cell->state);
+ switch (cell->state) {
+- case AFS_CELL_INACTIVE:
+- case AFS_CELL_FAILED:
+- down_write(&net->cells_lock);
+- active = 1;
+- if (atomic_try_cmpxchg_relaxed(&cell->active, &active, 0)) {
+- rb_erase(&cell->net_node, &net->cells);
+- trace_afs_cell(cell->debug_id, refcount_read(&cell->ref), 0,
+- afs_cell_trace_unuse_delete);
+- smp_store_release(&cell->state, AFS_CELL_REMOVED);
+- }
+- up_write(&net->cells_lock);
+- if (cell->state == AFS_CELL_REMOVED) {
+- wake_up_var(&cell->state);
+- goto final_destruction;
+- }
+- if (cell->state == AFS_CELL_FAILED)
+- goto done;
+- smp_store_release(&cell->state, AFS_CELL_UNSET);
+- wake_up_var(&cell->state);
+- goto again;
+-
+- case AFS_CELL_UNSET:
+- smp_store_release(&cell->state, AFS_CELL_ACTIVATING);
+- wake_up_var(&cell->state);
+- goto again;
+-
+- case AFS_CELL_ACTIVATING:
+- ret = afs_activate_cell(net, cell);
+- if (ret < 0)
+- goto activation_failed;
++ case AFS_CELL_SETTING_UP:
++ goto set_up_cell;
++ case AFS_CELL_UNLOOKED:
++ case AFS_CELL_ACTIVE:
++ goto cell_is_active;
++ case AFS_CELL_REMOVING:
++ WARN_ON_ONCE(1);
++ return false;
++ case AFS_CELL_DEAD:
++ return false;
++ default:
++ _debug("bad state %u", cell->state);
++ WARN_ON_ONCE(1); /* Unhandled state */
++ return false;
++ }
+
+- smp_store_release(&cell->state, AFS_CELL_ACTIVE);
+- wake_up_var(&cell->state);
+- goto again;
++set_up_cell:
++ ret = afs_activate_cell(net, cell);
++ if (ret < 0) {
++ cell->error = ret;
++ goto remove_cell;
++ }
+
+- case AFS_CELL_ACTIVE:
+- if (atomic_read(&cell->active) > 1) {
+- if (test_and_clear_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags)) {
+- ret = afs_update_cell(cell);
+- if (ret < 0)
+- cell->error = ret;
+- }
+- goto done;
+- }
+- smp_store_release(&cell->state, AFS_CELL_DEACTIVATING);
+- wake_up_var(&cell->state);
+- goto again;
++ afs_set_cell_state(cell, AFS_CELL_UNLOOKED);
+
+- case AFS_CELL_DEACTIVATING:
+- if (atomic_read(&cell->active) > 1)
+- goto reverse_deactivation;
+- afs_deactivate_cell(net, cell);
+- smp_store_release(&cell->state, AFS_CELL_INACTIVE);
+- wake_up_var(&cell->state);
+- goto again;
++cell_is_active:
++ if (afs_has_cell_expired(cell, &next_manage))
++ goto remove_cell;
+
+- case AFS_CELL_REMOVED:
+- goto done;
++ if (test_and_clear_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags)) {
++ ret = afs_update_cell(cell);
++ if (ret < 0)
++ cell->error = ret;
++ if (cell->state == AFS_CELL_UNLOOKED)
++ afs_set_cell_state(cell, AFS_CELL_ACTIVE);
++ }
+
+- default:
+- break;
++ if (next_manage < TIME64_MAX && cell->net->live) {
++ time64_t now = ktime_get_real_seconds();
++
++ if (next_manage - now <= 0)
++ afs_queue_cell(cell, afs_cell_trace_queue_again);
++ else
++ afs_set_cell_timer(cell, next_manage - now);
+ }
+- _debug("bad state %u", cell->state);
+- BUG(); /* Unhandled state */
++ _leave(" [done %u]", cell->state);
++ return false;
+
+-activation_failed:
+- cell->error = ret;
+- afs_deactivate_cell(net, cell);
++remove_cell:
++ down_write(&net->cells_lock);
+
+- smp_store_release(&cell->state, AFS_CELL_FAILED); /* vs error */
+- wake_up_var(&cell->state);
+- goto again;
++ if (atomic_read(&cell->active)) {
++ up_write(&net->cells_lock);
++ goto cell_is_active;
++ }
+
+-reverse_deactivation:
+- smp_store_release(&cell->state, AFS_CELL_ACTIVE);
+- wake_up_var(&cell->state);
+- _leave(" [deact->act]");
+- return;
++ /* Make sure that the expiring server records are going to see the fact
++ * that the cell is caput.
++ */
++ afs_set_cell_state(cell, AFS_CELL_REMOVING);
+
+-done:
+- _leave(" [done %u]", cell->state);
+- return;
++ afs_deactivate_cell(net, cell);
++ afs_purge_servers(cell);
++
++ rb_erase(&cell->net_node, &net->cells);
++ afs_see_cell(cell, afs_cell_trace_unuse_delete);
++ up_write(&net->cells_lock);
+
+-final_destruction:
+ /* The root volume is pinning the cell */
+ afs_put_volume(cell->root_volume, afs_volume_trace_put_cell_root);
+ cell->root_volume = NULL;
+- afs_put_cell(cell, afs_cell_trace_put_destroy);
++
++ afs_set_cell_state(cell, AFS_CELL_DEAD);
++ return true;
+ }
+
+ static void afs_manage_cell_work(struct work_struct *work)
+ {
+ struct afs_cell *cell = container_of(work, struct afs_cell, manager);
++ bool final_put;
+
+- afs_manage_cell(cell);
+- afs_put_cell(cell, afs_cell_trace_put_queue_work);
+-}
+-
+-/*
+- * Manage the records of cells known to a network namespace. This includes
+- * updating the DNS records and garbage collecting unused cells that were
+- * automatically added.
+- *
+- * Note that constructed cell records may only be removed from net->cells by
+- * this work item, so it is safe for this work item to stash a cursor pointing
+- * into the tree and then return to caller (provided it skips cells that are
+- * still under construction).
+- *
+- * Note also that we were given an increment on net->cells_outstanding by
+- * whoever queued us that we need to deal with before returning.
+- */
+-void afs_manage_cells(struct work_struct *work)
+-{
+- struct afs_net *net = container_of(work, struct afs_net, cells_manager);
+- struct rb_node *cursor;
+- time64_t now = ktime_get_real_seconds(), next_manage = TIME64_MAX;
+- bool purging = !net->live;
+-
+- _enter("");
+-
+- /* Trawl the cell database looking for cells that have expired from
+- * lack of use and cells whose DNS results have expired and dispatch
+- * their managers.
+- */
+- down_read(&net->cells_lock);
+-
+- for (cursor = rb_first(&net->cells); cursor; cursor = rb_next(cursor)) {
+- struct afs_cell *cell =
+- rb_entry(cursor, struct afs_cell, net_node);
+- unsigned active;
+- bool sched_cell = false;
+-
+- active = atomic_read(&cell->active);
+- trace_afs_cell(cell->debug_id, refcount_read(&cell->ref),
+- active, afs_cell_trace_manage);
+-
+- ASSERTCMP(active, >=, 1);
+-
+- if (purging) {
+- if (test_and_clear_bit(AFS_CELL_FL_NO_GC, &cell->flags)) {
+- active = atomic_dec_return(&cell->active);
+- trace_afs_cell(cell->debug_id, refcount_read(&cell->ref),
+- active, afs_cell_trace_unuse_pin);
+- }
+- }
+-
+- if (active == 1) {
+- struct afs_vlserver_list *vllist;
+- time64_t expire_at = cell->last_inactive;
+-
+- read_lock(&cell->vl_servers_lock);
+- vllist = rcu_dereference_protected(
+- cell->vl_servers,
+- lockdep_is_held(&cell->vl_servers_lock));
+- if (vllist->nr_servers > 0)
+- expire_at += afs_cell_gc_delay;
+- read_unlock(&cell->vl_servers_lock);
+- if (purging || expire_at <= now)
+- sched_cell = true;
+- else if (expire_at < next_manage)
+- next_manage = expire_at;
+- }
+-
+- if (!purging) {
+- if (test_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags))
+- sched_cell = true;
+- }
+-
+- if (sched_cell)
+- afs_queue_cell(cell, afs_cell_trace_get_queue_manage);
+- }
+-
+- up_read(&net->cells_lock);
+-
+- /* Update the timer on the way out. We have to pass an increment on
+- * cells_outstanding in the namespace that we are in to the timer or
+- * the work scheduler.
+- */
+- if (!purging && next_manage < TIME64_MAX) {
+- now = ktime_get_real_seconds();
+-
+- if (next_manage - now <= 0) {
+- if (queue_work(afs_wq, &net->cells_manager))
+- atomic_inc(&net->cells_outstanding);
+- } else {
+- afs_set_cell_timer(net, next_manage - now);
+- }
+- }
+-
+- afs_dec_cells_outstanding(net);
+- _leave(" [%d]", atomic_read(&net->cells_outstanding));
++ afs_see_cell(cell, afs_cell_trace_manage);
++ final_put = afs_manage_cell(cell);
++ afs_see_cell(cell, afs_cell_trace_managed);
++ if (final_put)
++ afs_put_cell(cell, afs_cell_trace_put_final);
+ }
+
+ /*
+@@ -949,6 +901,7 @@ void afs_manage_cells(struct work_struct *work)
+ void afs_cell_purge(struct afs_net *net)
+ {
+ struct afs_cell *ws;
++ struct rb_node *cursor;
+
+ _enter("");
+
+@@ -956,14 +909,21 @@ void afs_cell_purge(struct afs_net *net)
+ ws = rcu_replace_pointer(net->ws_cell, NULL,
+ lockdep_is_held(&net->cells_lock));
+ up_write(&net->cells_lock);
+- afs_unuse_cell(net, ws, afs_cell_trace_unuse_ws);
++ afs_unuse_cell(ws, afs_cell_trace_unuse_ws);
++
++ _debug("kick cells");
++ down_read(&net->cells_lock);
++ for (cursor = rb_first(&net->cells); cursor; cursor = rb_next(cursor)) {
++ struct afs_cell *cell = rb_entry(cursor, struct afs_cell, net_node);
+
+- _debug("del timer");
+- if (del_timer_sync(&net->cells_timer))
+- atomic_dec(&net->cells_outstanding);
++ afs_see_cell(cell, afs_cell_trace_purge);
+
+- _debug("kick mgr");
+- afs_queue_cell_manager(net);
++ if (test_and_clear_bit(AFS_CELL_FL_NO_GC, &cell->flags))
++ afs_unuse_cell(cell, afs_cell_trace_unuse_pin);
++
++ afs_queue_cell(cell, afs_cell_trace_queue_purge);
++ }
++ up_read(&net->cells_lock);
+
+ _debug("wait");
+ wait_var_event(&net->cells_outstanding,
+diff --git a/fs/afs/cmservice.c b/fs/afs/cmservice.c
+index 0638cab00c6182..435f43db8490de 100644
+--- a/fs/afs/cmservice.c
++++ b/fs/afs/cmservice.c
+@@ -138,49 +138,6 @@ bool afs_cm_incoming_call(struct afs_call *call)
+ }
+ }
+
+-/*
+- * Find the server record by peer address and record a probe to the cache
+- * manager from a server.
+- */
+-static int afs_find_cm_server_by_peer(struct afs_call *call)
+-{
+- struct sockaddr_rxrpc srx;
+- struct afs_server *server;
+- struct rxrpc_peer *peer;
+-
+- peer = rxrpc_kernel_get_call_peer(call->net->socket, call->rxcall);
+-
+- server = afs_find_server(call->net, peer);
+- if (!server) {
+- trace_afs_cm_no_server(call, &srx);
+- return 0;
+- }
+-
+- call->server = server;
+- return 0;
+-}
+-
+-/*
+- * Find the server record by server UUID and record a probe to the cache
+- * manager from a server.
+- */
+-static int afs_find_cm_server_by_uuid(struct afs_call *call,
+- struct afs_uuid *uuid)
+-{
+- struct afs_server *server;
+-
+- rcu_read_lock();
+- server = afs_find_server_by_uuid(call->net, call->request);
+- rcu_read_unlock();
+- if (!server) {
+- trace_afs_cm_no_server_u(call, call->request);
+- return 0;
+- }
+-
+- call->server = server;
+- return 0;
+-}
+-
+ /*
+ * Clean up a cache manager call.
+ */
+@@ -322,10 +279,7 @@ static int afs_deliver_cb_callback(struct afs_call *call)
+
+ if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING))
+ return afs_io_error(call, afs_io_error_cm_reply);
+-
+- /* we'll need the file server record as that tells us which set of
+- * vnodes to operate upon */
+- return afs_find_cm_server_by_peer(call);
++ return 0;
+ }
+
+ /*
+@@ -349,18 +303,10 @@ static void SRXAFSCB_InitCallBackState(struct work_struct *work)
+ */
+ static int afs_deliver_cb_init_call_back_state(struct afs_call *call)
+ {
+- int ret;
+-
+ _enter("");
+
+ afs_extract_discard(call, 0);
+- ret = afs_extract_data(call, false);
+- if (ret < 0)
+- return ret;
+-
+- /* we'll need the file server record as that tells us which set of
+- * vnodes to operate upon */
+- return afs_find_cm_server_by_peer(call);
++ return afs_extract_data(call, false);
+ }
+
+ /*
+@@ -373,8 +319,6 @@ static int afs_deliver_cb_init_call_back_state3(struct afs_call *call)
+ __be32 *b;
+ int ret;
+
+- _enter("");
+-
+ _enter("{%u}", call->unmarshall);
+
+ switch (call->unmarshall) {
+@@ -420,9 +364,23 @@ static int afs_deliver_cb_init_call_back_state3(struct afs_call *call)
+ if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING))
+ return afs_io_error(call, afs_io_error_cm_reply);
+
+- /* we'll need the file server record as that tells us which set of
+- * vnodes to operate upon */
+- return afs_find_cm_server_by_uuid(call, call->request);
++ if (!call->server) {
++ trace_afs_cm_no_server_u(call, call->request);
++ return 0;
++ }
++
++ if (!call->server) {
++ trace_afs_cm_no_server_u(call, call->request);
++ return 0;
++ }
++
++ if (memcmp(call->request, &call->server->_uuid, sizeof(call->server->_uuid)) != 0) {
++ pr_notice("Callback UUID does not match fileserver UUID\n");
++ trace_afs_cm_no_server_u(call, call->request);
++ return 0;
++ }
++
++ return 0;
+ }
+
+ /*
+@@ -454,7 +412,7 @@ static int afs_deliver_cb_probe(struct afs_call *call)
+
+ if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING))
+ return afs_io_error(call, afs_io_error_cm_reply);
+- return afs_find_cm_server_by_peer(call);
++ return 0;
+ }
+
+ /*
+@@ -531,7 +489,7 @@ static int afs_deliver_cb_probe_uuid(struct afs_call *call)
+
+ if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING))
+ return afs_io_error(call, afs_io_error_cm_reply);
+- return afs_find_cm_server_by_peer(call);
++ return 0;
+ }
+
+ /*
+@@ -591,7 +549,7 @@ static int afs_deliver_cb_tell_me_about_yourself(struct afs_call *call)
+
+ if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING))
+ return afs_io_error(call, afs_io_error_cm_reply);
+- return afs_find_cm_server_by_peer(call);
++ return 0;
+ }
+
+ /*
+@@ -665,9 +623,5 @@ static int afs_deliver_yfs_cb_callback(struct afs_call *call)
+
+ if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING))
+ return afs_io_error(call, afs_io_error_cm_reply);
+-
+- /* We'll need the file server record as that tells us which set of
+- * vnodes to operate upon.
+- */
+- return afs_find_cm_server_by_peer(call);
++ return 0;
+ }
+diff --git a/fs/afs/dynroot.c b/fs/afs/dynroot.c
+index 17dffa3b27e571..8beac43c21ce2e 100644
+--- a/fs/afs/dynroot.c
++++ b/fs/afs/dynroot.c
+@@ -108,7 +108,9 @@ static struct dentry *afs_dynroot_lookup_cell(struct inode *dir, struct dentry *
+ dotted = true;
+ }
+
+- cell = afs_lookup_cell(net, name, len, NULL, false);
++ cell = afs_lookup_cell(net, name, len, NULL,
++ AFS_LOOKUP_CELL_DYNROOT,
++ afs_cell_trace_use_lookup_dynroot);
+ if (IS_ERR(cell)) {
+ ret = PTR_ERR(cell);
+ goto out_no_cell;
+@@ -124,7 +126,7 @@ static struct dentry *afs_dynroot_lookup_cell(struct inode *dir, struct dentry *
+ return d_splice_alias(inode, dentry);
+
+ out:
+- afs_unuse_cell(cell->net, cell, afs_cell_trace_unuse_lookup_dynroot);
++ afs_unuse_cell(cell, afs_cell_trace_unuse_lookup_dynroot);
+ out_no_cell:
+ if (!inode)
+ return d_splice_alias(inode, dentry);
+@@ -166,7 +168,7 @@ static void afs_dynroot_d_release(struct dentry *dentry)
+ {
+ struct afs_cell *cell = dentry->d_fsdata;
+
+- afs_unuse_cell(cell->net, cell, afs_cell_trace_unuse_dynroot_mntpt);
++ afs_unuse_cell(cell, afs_cell_trace_unuse_dynroot_mntpt);
+ }
+
+ /*
+@@ -276,7 +278,7 @@ static struct dentry *afs_lookup_atcell(struct inode *dir, struct dentry *dentry
+ }
+
+ /*
+- * Transcribe the cell database into readdir content under the RCU read lock.
++ * Transcribe the cell database into readdir content under net->cells_lock.
+ * Each cell produces two entries, one prefixed with a dot and one not.
+ */
+ static int afs_dynroot_readdir_cells(struct afs_net *net, struct dir_context *ctx)
+@@ -292,8 +294,8 @@ static int afs_dynroot_readdir_cells(struct afs_net *net, struct dir_context *ct
+ cell = idr_get_next(&net->cells_dyn_ino, &ix);
+ if (!cell)
+ return 0;
+- if (READ_ONCE(cell->state) == AFS_CELL_FAILED ||
+- READ_ONCE(cell->state) == AFS_CELL_REMOVED) {
++ if (READ_ONCE(cell->state) == AFS_CELL_REMOVING ||
++ READ_ONCE(cell->state) == AFS_CELL_DEAD) {
+ ctx->pos += 2;
+ ctx->pos &= ~1;
+ continue;
+diff --git a/fs/afs/fs_probe.c b/fs/afs/fs_probe.c
+index b516d05b0fefc5..07a8bfbdd9b9de 100644
+--- a/fs/afs/fs_probe.c
++++ b/fs/afs/fs_probe.c
+@@ -235,20 +235,20 @@ out:
+ * Probe all of a fileserver's addresses to find out the best route and to
+ * query its capabilities.
+ */
+-void afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server,
+- struct afs_addr_list *new_alist, struct key *key)
++int afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server,
++ struct afs_addr_list *new_alist, struct key *key)
+ {
+ struct afs_endpoint_state *estate, *old;
+- struct afs_addr_list *alist;
++ struct afs_addr_list *old_alist = NULL, *alist;
+ unsigned long unprobed;
+
+ _enter("%pU", &server->uuid);
+
+ estate = kzalloc(sizeof(*estate), GFP_KERNEL);
+ if (!estate)
+- return;
++ return -ENOMEM;
+
+- refcount_set(&estate->ref, 1);
++ refcount_set(&estate->ref, 2);
+ estate->server_id = server->debug_id;
+ estate->rtt = UINT_MAX;
+
+@@ -256,21 +256,31 @@ void afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server,
+
+ old = rcu_dereference_protected(server->endpoint_state,
+ lockdep_is_held(&server->fs_lock));
+- estate->responsive_set = old->responsive_set;
+- estate->addresses = afs_get_addrlist(new_alist ?: old->addresses,
+- afs_alist_trace_get_estate);
++ if (old) {
++ estate->responsive_set = old->responsive_set;
++ if (!new_alist)
++ new_alist = old->addresses;
++ }
++
++ if (old_alist != new_alist)
++ afs_set_peer_appdata(server, old_alist, new_alist);
++
++ estate->addresses = afs_get_addrlist(new_alist, afs_alist_trace_get_estate);
+ alist = estate->addresses;
+ estate->probe_seq = ++server->probe_counter;
+ atomic_set(&estate->nr_probing, alist->nr_addrs);
+
++ if (new_alist)
++ server->addr_version = new_alist->version;
+ rcu_assign_pointer(server->endpoint_state, estate);
+- set_bit(AFS_ESTATE_SUPERSEDED, &old->flags);
+ write_unlock(&server->fs_lock);
++ if (old)
++ set_bit(AFS_ESTATE_SUPERSEDED, &old->flags);
+
+ trace_afs_estate(estate->server_id, estate->probe_seq, refcount_read(&estate->ref),
+ afs_estate_trace_alloc_probe);
+
+- afs_get_address_preferences(net, alist);
++ afs_get_address_preferences(net, new_alist);
+
+ server->probed_at = jiffies;
+ unprobed = (1UL << alist->nr_addrs) - 1;
+@@ -293,6 +303,8 @@ void afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server,
+ }
+
+ afs_put_endpoint_state(old, afs_estate_trace_put_probe);
++ afs_put_endpoint_state(estate, afs_estate_trace_put_probe);
++ return 0;
+ }
+
+ /*
+diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c
+index 098fa034a1ccb4..1f8a7878e9d960 100644
+--- a/fs/afs/fsclient.c
++++ b/fs/afs/fsclient.c
+@@ -1650,7 +1650,7 @@ int afs_fs_give_up_all_callbacks(struct afs_net *net, struct afs_server *server,
+ bp = call->request;
+ *bp++ = htonl(FSGIVEUPALLCALLBACKS);
+
+- call->server = afs_use_server(server, afs_server_trace_give_up_cb);
++ call->server = afs_use_server(server, false, afs_server_trace_use_give_up_cb);
+ afs_make_call(call, GFP_NOFS);
+ afs_wait_for_call_to_complete(call);
+ ret = call->error;
+@@ -1756,7 +1756,7 @@ bool afs_fs_get_capabilities(struct afs_net *net, struct afs_server *server,
+ return false;
+
+ call->key = key;
+- call->server = afs_use_server(server, afs_server_trace_get_caps);
++ call->server = afs_use_server(server, false, afs_server_trace_use_get_caps);
+ call->peer = rxrpc_kernel_get_peer(estate->addresses->addrs[addr_index].peer);
+ call->probe = afs_get_endpoint_state(estate, afs_estate_trace_get_getcaps);
+ call->probe_index = addr_index;
+diff --git a/fs/afs/internal.h b/fs/afs/internal.h
+index f9b90e59659317..beaedbb3a01ce4 100644
+--- a/fs/afs/internal.h
++++ b/fs/afs/internal.h
+@@ -304,8 +304,6 @@ struct afs_net {
+ struct rb_root cells;
+ struct idr cells_dyn_ino; /* cell->dynroot_ino mapping */
+ struct afs_cell __rcu *ws_cell;
+- struct work_struct cells_manager;
+- struct timer_list cells_timer;
+ atomic_t cells_outstanding;
+ struct rw_semaphore cells_lock;
+ struct mutex cells_alias_lock;
+@@ -317,18 +315,11 @@ struct afs_net {
+ * cell, but in practice, people create aliases and subsets and there's
+ * no easy way to distinguish them.
+ */
+- seqlock_t fs_lock; /* For fs_servers, fs_probe_*, fs_proc */
+- struct rb_root fs_servers; /* afs_server (by server UUID or address) */
++ seqlock_t fs_lock; /* For fs_probe_*, fs_proc */
+ struct list_head fs_probe_fast; /* List of afs_server to probe at 30s intervals */
+ struct list_head fs_probe_slow; /* List of afs_server to probe at 5m intervals */
+ struct hlist_head fs_proc; /* procfs servers list */
+
+- struct hlist_head fs_addresses; /* afs_server (by lowest IPv6 addr) */
+- seqlock_t fs_addr_lock; /* For fs_addresses[46] */
+-
+- struct work_struct fs_manager;
+- struct timer_list fs_timer;
+-
+ struct work_struct fs_prober;
+ struct timer_list fs_probe_timer;
+ atomic_t servers_outstanding;
+@@ -361,13 +352,11 @@ struct afs_net {
+ extern const char afs_init_sysname[];
+
+ enum afs_cell_state {
+- AFS_CELL_UNSET,
+- AFS_CELL_ACTIVATING,
++ AFS_CELL_SETTING_UP,
++ AFS_CELL_UNLOOKED,
+ AFS_CELL_ACTIVE,
+- AFS_CELL_DEACTIVATING,
+- AFS_CELL_INACTIVE,
+- AFS_CELL_FAILED,
+- AFS_CELL_REMOVED,
++ AFS_CELL_REMOVING,
++ AFS_CELL_DEAD,
+ };
+
+ /*
+@@ -398,7 +387,9 @@ struct afs_cell {
+ struct afs_cell *alias_of; /* The cell this is an alias of */
+ struct afs_volume *root_volume; /* The root.cell volume if there is one */
+ struct key *anonymous_key; /* anonymous user key for this cell */
++ struct work_struct destroyer; /* Destroyer for cell */
+ struct work_struct manager; /* Manager for init/deinit/dns */
++ struct timer_list management_timer; /* General management timer */
+ struct hlist_node proc_link; /* /proc cell list link */
+ time64_t dns_expiry; /* Time AFSDB/SRV record expires */
+ time64_t last_inactive; /* Time of last drop of usage count */
+@@ -424,7 +415,7 @@ struct afs_cell {
+
+ /* Active fileserver interaction state. */
+ struct rb_root fs_servers; /* afs_server (by server UUID) */
+- seqlock_t fs_lock; /* For fs_servers */
++ struct rw_semaphore fs_lock; /* For fs_servers */
+
+ /* VL server list. */
+ rwlock_t vl_servers_lock; /* Lock on vl_servers */
+@@ -432,6 +423,7 @@ struct afs_cell {
+
+ u8 name_len; /* Length of name */
+ char *name; /* Cell name, case-flattened and NUL-padded */
++ char *key_desc; /* Authentication key description */
+ };
+
+ /*
+@@ -559,22 +551,22 @@ struct afs_server {
+ };
+
+ struct afs_cell *cell; /* Cell to which belongs (pins ref) */
+- struct rb_node uuid_rb; /* Link in net->fs_servers */
+- struct afs_server __rcu *uuid_next; /* Next server with same UUID */
+- struct afs_server *uuid_prev; /* Previous server with same UUID */
+- struct list_head probe_link; /* Link in net->fs_probe_list */
+- struct hlist_node addr_link; /* Link in net->fs_addresses6 */
++ struct rb_node uuid_rb; /* Link in cell->fs_servers */
++ struct list_head probe_link; /* Link in net->fs_probe_* */
+ struct hlist_node proc_link; /* Link in net->fs_proc */
+ struct list_head volumes; /* RCU list of afs_server_entry objects */
+- struct afs_server *gc_next; /* Next server in manager's list */
++ struct work_struct destroyer; /* Work item to try and destroy a server */
++ struct timer_list timer; /* Management timer */
+ time64_t unuse_time; /* Time at which last unused */
+ unsigned long flags;
+ #define AFS_SERVER_FL_RESPONDING 0 /* The server is responding */
+ #define AFS_SERVER_FL_UPDATING 1
+ #define AFS_SERVER_FL_NEEDS_UPDATE 2 /* Fileserver address list is out of date */
+-#define AFS_SERVER_FL_NOT_READY 4 /* The record is not ready for use */
+-#define AFS_SERVER_FL_NOT_FOUND 5 /* VL server says no such server */
+-#define AFS_SERVER_FL_VL_FAIL 6 /* Failed to access VL server */
++#define AFS_SERVER_FL_UNCREATED 3 /* The record needs creating */
++#define AFS_SERVER_FL_CREATING 4 /* The record is being created */
++#define AFS_SERVER_FL_EXPIRED 5 /* The record has expired */
++#define AFS_SERVER_FL_NOT_FOUND 6 /* VL server says no such server */
++#define AFS_SERVER_FL_VL_FAIL 7 /* Failed to access VL server */
+ #define AFS_SERVER_FL_MAY_HAVE_CB 8 /* May have callbacks on this fileserver */
+ #define AFS_SERVER_FL_IS_YFS 16 /* Server is YFS not AFS */
+ #define AFS_SERVER_FL_NO_IBULK 17 /* Fileserver doesn't support FS.InlineBulkStatus */
+@@ -584,6 +576,7 @@ struct afs_server {
+ atomic_t active; /* Active user count */
+ u32 addr_version; /* Address list version */
+ u16 service_id; /* Service ID we're using. */
++ short create_error; /* Creation error */
+ unsigned int rtt; /* Server's current RTT in uS */
+ unsigned int debug_id; /* Debugging ID for traces */
+
+@@ -1004,6 +997,9 @@ extern int afs_merge_fs_addr4(struct afs_net *net, struct afs_addr_list *addr,
+ __be32 xdr, u16 port);
+ extern int afs_merge_fs_addr6(struct afs_net *net, struct afs_addr_list *addr,
+ __be32 *xdr, u16 port);
++void afs_set_peer_appdata(struct afs_server *server,
++ struct afs_addr_list *old_alist,
++ struct afs_addr_list *new_alist);
+
+ /*
+ * addr_prefs.c
+@@ -1040,16 +1036,26 @@ static inline bool afs_cb_is_broken(unsigned int cb_break,
+ extern int afs_cell_init(struct afs_net *, const char *);
+ extern struct afs_cell *afs_find_cell(struct afs_net *, const char *, unsigned,
+ enum afs_cell_trace);
+-extern struct afs_cell *afs_lookup_cell(struct afs_net *, const char *, unsigned,
+- const char *, bool);
++enum afs_lookup_cell_for {
++ AFS_LOOKUP_CELL_DYNROOT,
++ AFS_LOOKUP_CELL_MOUNTPOINT,
++ AFS_LOOKUP_CELL_DIRECT_MOUNT,
++ AFS_LOOKUP_CELL_PRELOAD,
++ AFS_LOOKUP_CELL_ROOTCELL,
++ AFS_LOOKUP_CELL_ALIAS_CHECK,
++};
++struct afs_cell *afs_lookup_cell(struct afs_net *net,
++ const char *name, unsigned int namesz,
++ const char *vllist,
++ enum afs_lookup_cell_for reason,
++ enum afs_cell_trace trace);
+ extern struct afs_cell *afs_use_cell(struct afs_cell *, enum afs_cell_trace);
+-extern void afs_unuse_cell(struct afs_net *, struct afs_cell *, enum afs_cell_trace);
++void afs_unuse_cell(struct afs_cell *cell, enum afs_cell_trace reason);
+ extern struct afs_cell *afs_get_cell(struct afs_cell *, enum afs_cell_trace);
+ extern void afs_see_cell(struct afs_cell *, enum afs_cell_trace);
+ extern void afs_put_cell(struct afs_cell *, enum afs_cell_trace);
+ extern void afs_queue_cell(struct afs_cell *, enum afs_cell_trace);
+-extern void afs_manage_cells(struct work_struct *);
+-extern void afs_cells_timer(struct timer_list *);
++void afs_set_cell_timer(struct afs_cell *cell, unsigned int delay_secs);
+ extern void __net_exit afs_cell_purge(struct afs_net *);
+
+ /*
+@@ -1188,8 +1194,8 @@ struct afs_endpoint_state *afs_get_endpoint_state(struct afs_endpoint_state *est
+ enum afs_estate_trace where);
+ void afs_put_endpoint_state(struct afs_endpoint_state *estate, enum afs_estate_trace where);
+ extern void afs_fileserver_probe_result(struct afs_call *);
+-void afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server,
+- struct afs_addr_list *new_addrs, struct key *key);
++int afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server,
++ struct afs_addr_list *new_alist, struct key *key);
+ int afs_wait_for_fs_probes(struct afs_operation *op, struct afs_server_state *states, bool intr);
+ extern void afs_probe_fileserver(struct afs_net *, struct afs_server *);
+ extern void afs_fs_probe_dispatcher(struct work_struct *);
+@@ -1472,20 +1478,30 @@ extern void __exit afs_clean_up_permit_cache(void);
+ */
+ extern spinlock_t afs_server_peer_lock;
+
+-extern struct afs_server *afs_find_server(struct afs_net *, const struct rxrpc_peer *);
+-extern struct afs_server *afs_find_server_by_uuid(struct afs_net *, const uuid_t *);
++struct afs_server *afs_find_server(const struct rxrpc_peer *peer);
+ extern struct afs_server *afs_lookup_server(struct afs_cell *, struct key *, const uuid_t *, u32);
+ extern struct afs_server *afs_get_server(struct afs_server *, enum afs_server_trace);
+-extern struct afs_server *afs_use_server(struct afs_server *, enum afs_server_trace);
+-extern void afs_unuse_server(struct afs_net *, struct afs_server *, enum afs_server_trace);
+-extern void afs_unuse_server_notime(struct afs_net *, struct afs_server *, enum afs_server_trace);
++struct afs_server *afs_use_server(struct afs_server *server, bool activate,
++ enum afs_server_trace reason);
++void afs_unuse_server(struct afs_net *net, struct afs_server *server,
++ enum afs_server_trace reason);
++void afs_unuse_server_notime(struct afs_net *net, struct afs_server *server,
++ enum afs_server_trace reason);
+ extern void afs_put_server(struct afs_net *, struct afs_server *, enum afs_server_trace);
+-extern void afs_manage_servers(struct work_struct *);
+-extern void afs_servers_timer(struct timer_list *);
++void afs_purge_servers(struct afs_cell *cell);
+ extern void afs_fs_probe_timer(struct timer_list *);
+-extern void __net_exit afs_purge_servers(struct afs_net *);
++void __net_exit afs_wait_for_servers(struct afs_net *net);
+ bool afs_check_server_record(struct afs_operation *op, struct afs_server *server, struct key *key);
+
++static inline void afs_see_server(struct afs_server *server, enum afs_server_trace trace)
++{
++ int r = refcount_read(&server->ref);
++ int a = atomic_read(&server->active);
++
++ trace_afs_server(server->debug_id, r, a, trace);
++
++}
++
+ static inline void afs_inc_servers_outstanding(struct afs_net *net)
+ {
+ atomic_inc(&net->servers_outstanding);
+diff --git a/fs/afs/main.c b/fs/afs/main.c
+index 5fffbc4d47aff5..874e50bda8f858 100644
+--- a/fs/afs/main.c
++++ b/fs/afs/main.c
+@@ -78,24 +78,15 @@ static int __net_init afs_net_init(struct net *net_ns)
+ net->cells = RB_ROOT;
+ idr_init(&net->cells_dyn_ino);
+ init_rwsem(&net->cells_lock);
+- INIT_WORK(&net->cells_manager, afs_manage_cells);
+- timer_setup(&net->cells_timer, afs_cells_timer, 0);
+-
+ mutex_init(&net->cells_alias_lock);
+ mutex_init(&net->proc_cells_lock);
+ INIT_HLIST_HEAD(&net->proc_cells);
+
+ seqlock_init(&net->fs_lock);
+- net->fs_servers = RB_ROOT;
+ INIT_LIST_HEAD(&net->fs_probe_fast);
+ INIT_LIST_HEAD(&net->fs_probe_slow);
+ INIT_HLIST_HEAD(&net->fs_proc);
+
+- INIT_HLIST_HEAD(&net->fs_addresses);
+- seqlock_init(&net->fs_addr_lock);
+-
+- INIT_WORK(&net->fs_manager, afs_manage_servers);