diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e0193a18f4fad70bd61b187cf1f9c5434b8a4494..99d2596e0174d13ea9ac5f64ad3fbacccec87391 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5538,7 +5538,7 @@ rdt= [HW,X86,RDT] Turn on/off individual RDT features. List is: cmt, mbmtotal, mbmlocal, l3cat, l3cdp, l2cat, l2cdp, - mba, smba, bmec, abmc, sdciae. + mba, smba, bmec. E.g. to turn on cmt and turn off mba use: rdt=cmt,!mba diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index 253e9743de2f96de515ee844f7b3e3a671a368b1..44f9bd78539d3603bc9a31b19c90cc6f7c96c14c 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -152,8 +152,6 @@ infrastructure: +------------------------------+---------+---------+ | DIT | [51-48] | y | +------------------------------+---------+---------+ - | MPAM | [43-40] | n | - +------------------------------+---------+---------+ | SVE | [35-32] | y | +------------------------------+---------+---------+ | GIC | [27-24] | n | diff --git a/Documentation/arch/x86/index.rst b/Documentation/arch/x86/index.rst index 8ac64d7de4dc9a4a8ebaa58e189efea5e876ace5..00f9a99689fb80ea865385d786199efdb82d4571 100644 --- a/Documentation/arch/x86/index.rst +++ b/Documentation/arch/x86/index.rst @@ -31,7 +31,6 @@ x86-specific Documentation pti mds microcode - resctrl tsx_async_abort buslock usb-legacy-support diff --git a/Documentation/devicetree/bindings/arm/arm,mpam-msc.yaml b/Documentation/devicetree/bindings/arm/arm,mpam-msc.yaml deleted file mode 100644 index 9d542ecb1a7d6c3e3ec820179de29f1dd4259bb4..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/arm/arm,mpam-msc.yaml +++ /dev/null @@ -1,227 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/arm/arm,mpam-msc.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Arm Memory System Resource Partitioning and Monitoring (MPAM) - -description: | - The Arm MPAM specification can be found here: - - https://developer.arm.com/documentation/ddi0598/latest - -maintainers: - - Rob Herring - -properties: - compatible: - items: - - const: arm,mpam-msc # Further details are discoverable - - const: arm,mpam-memory-controller-msc - - reg: - maxItems: 1 - description: A memory region containing registers as defined in the MPAM - specification. - - interrupts: - minItems: 1 - items: - - description: error (optional) - - description: overflow (optional, only for monitoring) - - interrupt-names: - oneOf: - - items: - - enum: [ error, overflow ] - - items: - - const: error - - const: overflow - - arm,not-ready-us: - description: The maximum time in microseconds for monitoring data to be - accurate after a settings change. For more information, see the - Not-Ready (NRDY) bit description in the MPAM specification. - - numa-node-id: true # see NUMA binding - - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - -patternProperties: - '^ris@[0-9a-f]$': - type: object - additionalProperties: false - description: | - RIS nodes for each RIS in an MSC. These nodes are required for each RIS - implementing known MPAM controls - - properties: - compatible: - enum: - # Bulk storage for cache - - arm,mpam-cache - # Memory bandwidth - - arm,mpam-memory - - reg: - minimum: 0 - maximum: 0xf - - cpus: - $ref: '/schemas/types.yaml#/definitions/phandle-array' - description: - Phandle(s) to the CPU node(s) this RIS belongs to. By default, the parent - device's affinity is used. - - arm,mpam-device: - $ref: '/schemas/types.yaml#/definitions/phandle' - description: - By default, the MPAM enabled device associated with a RIS is the MSC's - parent node. It is possible for each RIS to be associated with different - devices in which case 'arm,mpam-device' should be used. - - required: - - compatible - - reg - -required: - - compatible - - reg - -dependencies: - interrupts: [ interrupt-names ] - -additionalProperties: false - -examples: - - | - /* - cpus { - cpu@0 { - next-level-cache = <&L2_0>; - }; - cpu@100 { - next-level-cache = <&L2_1>; - }; - }; - */ - L2_0: cache-controller-0 { - compatible = "cache"; - cache-level = <2>; - cache-unified; - next-level-cache = <&L3>; - - }; - - L2_1: cache-controller-1 { - compatible = "cache"; - cache-level = <2>; - cache-unified; - next-level-cache = <&L3>; - - }; - - L3: cache-controller@30000000 { - compatible = "arm,dsu-l3-cache", "cache"; - cache-level = <3>; - cache-unified; - - ranges = <0x0 0x30000000 0x800000>; - #address-cells = <1>; - #size-cells = <1>; - - msc@10000 { - compatible = "arm,mpam-msc"; - - /* CPU affinity implied by parent cache node's */ - reg = <0x10000 0x2000>; - interrupts = <1>, <2>; - interrupt-names = "error", "overflow"; - arm,not-ready-us = <1>; - }; - }; - - mem: memory-controller@20000 { - compatible = "foo,a-memory-controller"; - reg = <0x20000 0x1000>; - - #address-cells = <1>; - #size-cells = <1>; - ranges; - - msc@21000 { - compatible = "arm,mpam-memory-controller-msc", "arm,mpam-msc"; - reg = <0x21000 0x1000>; - interrupts = <3>; - interrupt-names = "error"; - arm,not-ready-us = <1>; - numa-node-id = <1>; - }; - }; - - iommu@40000 { - reg = <0x40000 0x1000>; - - ranges; - #address-cells = <1>; - #size-cells = <1>; - - msc@41000 { - compatible = "arm,mpam-msc"; - reg = <0 0x1000>; - interrupts = <5>, <6>; - interrupt-names = "error", "overflow"; - arm,not-ready-us = <1>; - - #address-cells = <1>; - #size-cells = <0>; - - ris@2 { - compatible = "arm,mpam-cache"; - reg = <0>; - // TODO: How to map to device(s)? - }; - }; - }; - - msc@80000 { - compatible = "foo,a-standalone-msc"; - reg = <0x80000 0x1000>; - - clocks = <&clks 123>; - - ranges; - #address-cells = <1>; - #size-cells = <1>; - - msc@10000 { - compatible = "arm,mpam-msc"; - - reg = <0x10000 0x2000>; - interrupts = <7>; - interrupt-names = "overflow"; - arm,not-ready-us = <1>; - - #address-cells = <1>; - #size-cells = <0>; - - ris@0 { - compatible = "arm,mpam-cache"; - reg = <0>; - arm,mpam-device = <&L2_0>; - }; - - ris@1 { - compatible = "arm,mpam-memory"; - reg = <1>; - arm,mpam-device = <&mem>; - }; - }; - }; - -... diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst index 09cade7eaefc8c68e4e733ea011b345a4ce9fc9b..f9e388fd8d000298c53961a40afc52ccd5cbc25c 100644 --- a/Documentation/filesystems/index.rst +++ b/Documentation/filesystems/index.rst @@ -109,6 +109,7 @@ Documentation for filesystem implementations. qnx6 ramfs-rootfs-initramfs relay + resctrl romfs smb/index spufs/index diff --git a/Documentation/arch/x86/resctrl.rst b/Documentation/filesystems/resctrl.rst similarity index 83% rename from Documentation/arch/x86/resctrl.rst rename to Documentation/filesystems/resctrl.rst index 86adf6840bcd80e74ebb41d8cb712fc09208bc1a..361b524f6f3172d253d03c53d04a07ddaf93dffd 100644 --- a/Documentation/arch/x86/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -1,9 +1,9 @@ .. SPDX-License-Identifier: GPL-2.0 .. include:: -=========================================== -User Interface for Resource Control feature -=========================================== +===================================================== +User Interface for Resource Control feature (resctrl) +===================================================== :Copyright: |copy| 2016 Intel Corporation :Authors: - Fenghua Yu @@ -17,18 +17,16 @@ AMD refers to this feature as AMD Platform Quality of Service(AMD QoS). This feature is enabled by the CONFIG_X86_CPU_RESCTRL and the x86 /proc/cpuinfo flag bits: -=============================================================== ================================ -RDT (Resource Director Technology) Allocation "rdt_a" -CAT (Cache Allocation Technology) "cat_l3", "cat_l2" -CDP (Code and Data Prioritization) "cdp_l3", "cdp_l2" -CQM (Cache QoS Monitoring) "cqm_llc", "cqm_occup_llc" -MBM (Memory Bandwidth Monitoring) "cqm_mbm_total", "cqm_mbm_local" -MBA (Memory Bandwidth Allocation) "mba" -SMBA (Slow Memory Bandwidth Allocation) "" -BMEC (Bandwidth Monitoring Event Configuration) "" -ABMC (Assignable Bandwidth Monitoring Counters) "" -SDCIAE (Smart Data Cache Injection Allocation Enforcement) "" -=============================================================== ================================ +=============================================== ================================ +RDT (Resource Director Technology) Allocation "rdt_a" +CAT (Cache Allocation Technology) "cat_l3", "cat_l2" +CDP (Code and Data Prioritization) "cdp_l3", "cdp_l2" +CQM (Cache QoS Monitoring) "cqm_llc", "cqm_occup_llc" +MBM (Memory Bandwidth Monitoring) "cqm_mbm_total", "cqm_mbm_local" +MBA (Memory Bandwidth Allocation) "mba" +SMBA (Slow Memory Bandwidth Allocation) "" +BMEC (Bandwidth Monitoring Event Configuration) "" +=============================================== ================================ Historically, new features were made visible by default in /proc/cpuinfo. This resulted in the feature flags becoming hard to parse by humans. Adding a new @@ -47,7 +45,7 @@ mount options are: Enable code/data prioritization in L2 cache allocations. "mba_MBps": Enable the MBA Software Controller(mba_sc) to specify MBA - bandwidth in MBps + bandwidth in MiBps "debug": Make debug files accessible. Available debug files are annotated with "Available only with debug option". @@ -73,11 +71,6 @@ The 'info' directory contains information about the enabled resources. Each resource has its own subdirectory. The subdirectory names reflect the resource names. -Most of the files in the resource's subdirectory are read-only, and -describe properties of the resource. Resources that support global -configuration options also include writable files that can be used -to modify those settings. - Each subdirectory contains the following files with respect to allocation: @@ -142,77 +135,6 @@ related to allocation: "1": Non-contiguous 1s value in CBM is supported. -"io_alloc": - "io_alloc" enables system software to configure the portion of - the cache allocated for I/O traffic. File may only exist if the - system supports this feature on some of its cache resources. - - "disabled": - Resource supports "io_alloc" but the feature is disabled. - Portions of cache used for allocation of I/O traffic cannot - be configured. - "enabled": - Portions of cache used for allocation of I/O traffic - can be configured using "io_alloc_cbm". - "not supported": - Support not available for this resource. - - The feature can be modified by writing to the interface, for example: - - To enable:: - - # echo 1 > /sys/fs/resctrl/info/L3/io_alloc - - To disable:: - - # echo 0 > /sys/fs/resctrl/info/L3/io_alloc - - The underlying implementation may reduce resources available to - general (CPU) cache allocation. See architecture specific notes - below. Depending on usage requirements the feature can be enabled - or disabled. - - On AMD systems, io_alloc feature is supported by the L3 Smart - Data Cache Injection Allocation Enforcement (SDCIAE). The CLOSID for - io_alloc is the highest CLOSID supported by the resource. When - io_alloc is enabled, the highest CLOSID is dedicated to io_alloc and - no longer available for general (CPU) cache allocation. When CDP is - enabled, io_alloc routes I/O traffic using the highest CLOSID allocated - for the instruction cache (CDP_CODE), making this CLOSID no longer - available for general (CPU) cache allocation for both the CDP_CODE - and CDP_DATA resources. - -"io_alloc_cbm": - Capacity bitmasks that describe the portions of cache instances to - which I/O traffic from supported I/O devices are routed when "io_alloc" - is enabled. - - CBMs are displayed in the following format: - - =;=;... - - Example:: - - # cat /sys/fs/resctrl/info/L3/io_alloc_cbm - 0=ffff;1=ffff - - CBMs can be configured by writing to the interface. - - Example:: - - # echo 1=ff > /sys/fs/resctrl/info/L3/io_alloc_cbm - # cat /sys/fs/resctrl/info/L3/io_alloc_cbm - 0=ffff;1=00ff - - # echo "0=ff;1=f" > /sys/fs/resctrl/info/L3/io_alloc_cbm - # cat /sys/fs/resctrl/info/L3/io_alloc_cbm - 0=00ff;1=000f - - When CDP is enabled "io_alloc_cbm" associated with the CDP_DATA and CDP_CODE - resources may reflect the same values. For example, values read from and - written to /sys/fs/resctrl/info/L3DATA/io_alloc_cbm may be reflected by - /sys/fs/resctrl/info/L3CODE/io_alloc_cbm and vice versa. - Memory bandwidth(MB) subdirectory contains the following files with respect to allocation: @@ -334,205 +256,6 @@ with the following files: # cat /sys/fs/resctrl/info/L3_MON/mbm_local_bytes_config 0=0x30;1=0x30;3=0x15;4=0x15 -"mbm_mode": - Reports the list of assignable monitoring features supported. The - enclosed brackets indicate which feature is enabled. - :: - - cat /sys/fs/resctrl/info/L3_MON/mbm_mode - [mbm_cntr_assign] - legacy - - "mbm_cntr_assign": - AMD's ABMC feature is one of the mbm_cntr_assign mode supported. - The bandwidth monitoring feature on AMD system only guarantees - that RMIDs currently assigned to a processor will be tracked by - hardware. The counters of any other RMIDs which are no longer - being tracked will be reset to zero. The MBM event counters - return "Unavailable" for the RMIDs that are not tracked by - hardware. So, there can be only limited number of groups that can - give guaranteed monitoring numbers. With ever changing configurations - there is no way to definitely know which of these groups are being - tracked for certain point of time. Users do not have the option to - monitor a group or set of groups for certain period of time without - worrying about RMID being reset in between. - - The ABMC feature provides an option to the user to assign a hardware - counter to an RMID and monitor the bandwidth as long as it is assigned. - The assigned RMID will be tracked by the hardware until the user - unassigns it manually. There is no need to worry about counters being - reset during this period. - - "Legacy": - Legacy mode works without the assignment option. The monitoring works - as long as there are enough RMID counters available to support number - of monitoring groups. - - * To enable ABMC feature: - :: - - # echo "mbm_cntr_assign" > /sys/fs/resctrl/info/L3_MON/mbm_mode - - * To enable the legacy monitoring feature: - :: - - # echo "legacy" > /sys/fs/resctrl/info/L3_MON/mbm_mode - - The MBM event counters will reset when mbm_mode is changed. Moving to - mbm_cntr_assign will require users to assign the counters to the events to - read the events. Otherwise, the MBM event counters will return "Unassigned" - when read. - -"num_mbm_cntrs": - The number of monitoring counters available for assignment. - - Resctrl subsystem provides the interface to count maximum of two - MBM events per group, from a combination of total and local events. - Keeping the current interface, users can assign a maximum of two - monitoring counters per group. User will also have the option to - enable only one counter to the group. - - With limited number of counters, system can run out of assignable counters. - In mbm_cntr_assign mode, the MBM event counters will return "Unassigned" if - the counter is not assigned to the event when read. Users need to assign a - counter manually to read the events. - -"mbm_control": - Reports the resctrl group and monitor status of each group. - - List follows the following format: - "//=" - - Format for specific type of groups: - - * Default CTRL_MON group: - "//=" - - * Non-default CTRL_MON group: - "//=" - - * Child MON group of default CTRL_MON group: - "//=" - - * Child MON group of non-default CTRL_MON group: - "//=" - - Flags can be one of the following: - :: - - t MBM total event is enabled. - l MBM local event is enabled. - tl Both total and local MBM events are enabled. - _ None of the MBM events are enabled. Only works with opcode '=' for write. - - Examples: - :: - - # mkdir /sys/fs/resctrl/mon_groups/child_default_mon_grp - # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp - # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp/mon_groups/child_non_default_mon_grp - - # cat /sys/fs/resctrl/info/L3_MON/mbm_control - non_default_ctrl_mon_grp//0=tl;1=tl; - non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=tl; - //0=tl;1=tl; - /child_default_mon_grp/0=tl;1=tl; - - There are four resctrl groups. All the groups have total and local MBM events - enabled on domain 0 and 1. - - Assignment state can be updated by writing to the interface. - - Format is similar to the list format with addition of opcode for the - assignment operation. - - "//" - - Format for each type of groups: - - * Default CTRL_MON group: - "//" - - * Non-default CTRL_MON group: - "//" - - * Child MON group of default CTRL_MON group: - "//" - - * Child MON group of non-default CTRL_MON group: - "//" - - Domain_id '*' wil apply the flags on all the domains. - - Opcode can be one of the following: - :: - - = Update the assignment to match the MBM event. - + Assign a MBM event. - - Unassign a MBM event. - - Examples: - :: - - Initial group status: - # cat /sys/fs/resctrl/info/L3_MON/mbm_control - non_default_ctrl_mon_grp//0=tl;1=tl; - non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=tl; - //0=tl;1=tl; - /child_default_mon_grp/0=tl;1=tl; - - To update the default group to assign only total MBM event on domain 0: - # echo "//0=t" > /sys/fs/resctrl/info/L3_MON/mbm_control - - Assignment status after the update: - # cat /sys/fs/resctrl/info/L3_MON/mbm_control - non_default_ctrl_mon_grp//0=tl;1=tl; - non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=tl; - //0=t;1=tl; - /child_default_mon_grp/0=tl;1=tl; - - To update the MON group child_default_mon_grp to remove total MBM event on domain 1: - # echo "/child_default_mon_grp/1-t" > /sys/fs/resctrl/info/L3_MON/mbm_control - - Assignment status after the update: - $ cat /sys/fs/resctrl/info/L3_MON/mbm_control - non_default_ctrl_mon_grp//0=tl;1=tl; - non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=tl; - //0=t;1=tl; - /child_default_mon_grp/0=tl;1=l; - - To update the MON group non_default_ctrl_mon_grp/child_non_default_mon_grp to - unassign both local and total MBM events on domain 1: - # echo "non_default_ctrl_mon_grp/child_non_default_mon_grp/1=_" > - /sys/fs/resctrl/info/L3_MON/mbm_control - - Assignment status after the update: - non_default_ctrl_mon_grp//0=tl;1=tl; - non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=_; - //0=t;1=tl; - /child_default_mon_grp/0=tl;1=l; - - To update the default group to add a local MBM event domain 0. - # echo "//0+l" > /sys/fs/resctrl/info/L3_MON/mbm_control - - Assignment status after the update: - # cat /sys/fs/resctrl/info/L3_MON/mbm_control - non_default_ctrl_mon_grp//0=tl;1=tl; - non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=_; - //0=tl;1=tl; - /child_default_mon_grp/0=tl;1=l; - - To update the non default CTRL_MON group non_default_ctrl_mon_grp to unassign all - the MBM events on all the domains. - # echo "non_default_ctrl_mon_grp//*=_" > /sys/fs/resctrl/info/L3_MON/mbm_control - - Assignment status after the update: - #cat /sys/fs/resctrl/info/L3_MON/mbm_control - non_default_ctrl_mon_grp//0=_;1=_; - non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=_; - //0=tl;1=tl; - /child_default_mon_grp/0=tl;1=l; - "max_threshold_occupancy": Read/write file provides the largest value (in bytes) at which a previously used LLC_occupancy @@ -652,11 +375,25 @@ When monitoring is enabled all MON groups will also contain: all tasks in the group. In CTRL_MON groups these files provide the sum for all tasks in the CTRL_MON group and all tasks in MON groups. Please see example section for more details on usage. + On systems with Sub-NUMA Cluster (SNC) enabled there are extra + directories for each node (located within the "mon_L3_XX" directory + for the L3 cache they occupy). These are named "mon_sub_L3_YY" + where "YY" is the node number. "mon_hw_id": Available only with debug option. The identifier used by hardware for the monitor group. On x86 this is the RMID. +When the "mba_MBps" mount option is used all CTRL_MON groups will also contain: + +"mba_MBps_event": + Reading this file shows which memory bandwidth event is used + as input to the software feedback loop that keeps memory bandwidth + below the value specified in the schemata file. Writing the + name of one of the supported memory bandwidth events found in + /sys/fs/resctrl/info/L3_MON/mon_features changes the input + event. + Resource allocation rules ------------------------- @@ -723,6 +460,12 @@ during mkdir. max_threshold_occupancy is a user configurable value to determine the occupancy at which an RMID can be freed. +The mon_llc_occupancy_limbo tracepoint gives the precise occupancy in bytes +for a subset of RMID that are not immediately available for allocation. +This can't be relied on to produce output every second, it may be necessary +to attempt to create an empty monitor group to force an update. Output may +only be produced if creation of a control or monitor group fails. + Schemata files - general concepts --------------------------------- Each line in the file describes one resource. The line starts with @@ -755,6 +498,29 @@ if non-contiguous 1s value is supported. On a system with a 20-bit mask each bit represents 5% of the capacity of the cache. You could partition the cache into four equal parts with masks: 0x1f, 0x3e0, 0x7c00, 0xf8000. +Notes on Sub-NUMA Cluster mode +============================== +When SNC mode is enabled, Linux may load balance tasks between Sub-NUMA +nodes much more readily than between regular NUMA nodes since the CPUs +on Sub-NUMA nodes share the same L3 cache and the system may report +the NUMA distance between Sub-NUMA nodes with a lower value than used +for regular NUMA nodes. + +The top-level monitoring files in each "mon_L3_XX" directory provide +the sum of data across all SNC nodes sharing an L3 cache instance. +Users who bind tasks to the CPUs of a specific Sub-NUMA node can read +the "llc_occupancy", "mbm_total_bytes", and "mbm_local_bytes" in the +"mon_sub_L3_YY" directories to get node local data. + +Memory bandwidth allocation is still performed at the L3 cache +level. I.e. throttling controls are applied to all SNC nodes. + +L3 cache allocation bitmaps also apply to all SNC nodes. But note that +the amount of L3 cache represented by each bit is divided by the number +of SNC nodes per L3 cache. E.g. with a 100MB cache on a system with 10-bit +allocation masks each bit normally represents 10MB. With SNC mode enabled +with two SNC nodes per L3 cache, each bit only represents 5MB. + Memory bandwidth Allocation and monitoring ========================================== @@ -803,7 +569,7 @@ threads start using more cores in an rdtgroup, the actual bandwidth may increase or vary although user specified bandwidth percentage is same. In order to mitigate this and make the interface more user friendly, -resctrl added support for specifying the bandwidth in MBps as well. The +resctrl added support for specifying the bandwidth in MiBps as well. The kernel underneath would use a software feedback mechanism or a "Software Controller(mba_sc)" which reads the actual bandwidth using MBM counters and adjust the memory bandwidth percentages to ensure:: @@ -850,13 +616,13 @@ Memory b/w domain is L3 cache. MB:=bandwidth0;=bandwidth1;... -Memory bandwidth Allocation specified in MBps +Memory bandwidth Allocation specified in MiBps --------------------------------------------- Memory bandwidth domain is L3 cache. :: - MB:=bw_MBps0;=bw_MBps1;... + MB:=bw_MiBps0;=bw_MiBps1;... Slow Memory Bandwidth Allocation (SMBA) --------------------------------------- diff --git a/MAINTAINERS b/MAINTAINERS index 2def3d5f43b36253ecc6a02708dbed6d1e3b0a18..019464f29333d6587cb52deb32cd9b94c922844c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18199,9 +18199,11 @@ F: net/rds/ RDT - RESOURCE ALLOCATION M: Fenghua Yu M: Reinette Chatre +R: Dave Martin +R: James Morse L: linux-kernel@vger.kernel.org S: Supported -F: Documentation/arch/x86/resctrl* +F: Documentation/filesystems/resctrl.rst F: arch/x86/include/asm/resctrl.h F: arch/x86/kernel/cpu/resctrl/ F: fs/resctrl/ diff --git a/anolis/configs/L0-MANDATORY/arm64/CONFIG_ARM64_MPAM b/anolis/configs/L0-MANDATORY/arm64/CONFIG_ARM64_MPAM deleted file mode 100644 index 45957b7b4ea21fbd49e5e57512cad210afbdd776..0000000000000000000000000000000000000000 --- a/anolis/configs/L0-MANDATORY/arm64/CONFIG_ARM64_MPAM +++ /dev/null @@ -1 +0,0 @@ -CONFIG_ARM64_MPAM=y diff --git a/anolis/configs/L2-OPTIONAL/arm64/CONFIG_ACPI_MPAM b/anolis/configs/L2-OPTIONAL/arm64/CONFIG_ACPI_MPAM deleted file mode 100644 index e93cbd36cedc1b687c9c2d09124388fc11798580..0000000000000000000000000000000000000000 --- a/anolis/configs/L2-OPTIONAL/arm64/CONFIG_ACPI_MPAM +++ /dev/null @@ -1 +0,0 @@ -CONFIG_ACPI_MPAM=y diff --git a/arch/Kconfig b/arch/Kconfig index b2ccbaa3780e9177e7ecac9916c233e66e564513..7d35b843620cc5fc3749a2cafdbdacae637cf93f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1344,10 +1344,10 @@ config ARCH_HAS_PHYS_TO_DMA config ARCH_HAS_CPU_RESCTRL bool help - The 'resctrl' filesystem allows CPU controls of shared resources - such as caches and memory bandwidth to be configured. An architecture - selects this if it provides the arch-specific hooks for the filesystem - and needs the per-task CLOSID/RMID properties. + An architecture selects this option to indicate that the necessary + hooks are provided to support the common memory system usage + monitoring and control interfaces provided by the 'resctrl' + filesystem (see RESCTRL_FS). config HAVE_ARCH_COMPILER_H bool diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 592ea627edb397b34ed0a0557a0b998caad2bcb5..acedee7c16678002ff786e83027ec9a6e4a3f3cd 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2066,27 +2066,7 @@ config ARM64_TLB_RANGE The feature introduces new assembly instructions, and they were support when binutils >= 2.30. -config ARM64_MPAM - bool "Enable support for MPAM" - select ACPI_MPAM if ACPI - select ARCH_HAS_CPU_RESCTRL - select RESCTRL_FS - help - Memory Partitioning and Monitoring is an optional extension - that allows the CPUs to mark load and store transactions with - labels for partition-id and performance-monitoring-group. - System components, such as the caches, can use the partition-id - to apply a performance policy. MPAM monitors can use the - partition-id and performance-monitoring-group to measure the - cache occupancy or data throughput. - - Use of this extension requires CPU support, support in the - memory system components (MSC), and a description from firmware - of where the MSC are in the address space. - - MPAM is exposed to user-space via the resctrl pseudo filesystem. - -endmenu +endmenu # "ARMv8.4 architectural features" menu "ARMv8.5 architectural features" diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 385a6c162c32a992bbdb5885a61581b36577d50a..4dd7dce1917bc5d93217123462c56cb3ce945c6c 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -619,13 +619,6 @@ static inline bool id_aa64pfr1_sme(u64 pfr1) return val > 0; } -static inline bool id_aa64pfr0_mpam(u64 pfr0) -{ - u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT); - - return val > 0; -} - static inline bool id_aa64pfr1_mte(u64 pfr1) { u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT); diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index d93c2b5b0b59de1a6c7849c225a4fa948396e5a6..44336899ca278a643d49741b6439c842d00848da 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -274,21 +274,6 @@ msr spsr_el2, x0 .endm -.macro __init_el2_mpam -#ifdef CONFIG_ARM64_MPAM - /* Memory Partioning And Monitoring: disable EL2 traps */ - mrs x1, id_aa64pfr0_el1 - ubfx x0, x1, #ID_AA64PFR0_EL1_MPAM_SHIFT, #4 - cbz x0, .Lskip_mpam_\@ // skip if no MPAM - msr_s SYS_MPAM2_EL2, xzr // use the default partition - // and disable lower traps - mrs_s x0, SYS_MPAMIDR_EL1 - tbz x0, #17, .Lskip_mpam_\@ // skip if no MPAMHCR reg - msr_s SYS_MPAMHCR_EL2, xzr // clear TRAP_MPAMIDR_EL1 -> EL2 -.Lskip_mpam_\@: -#endif /* CONFIG_ARM64_MPAM */ -.endm - /** * Initialize EL2 registers to sane values. This should be called early on all * cores that were booted in EL2. Note that everything gets initialised as @@ -307,7 +292,6 @@ __init_el2_stage2 __init_el2_gicv3 __init_el2_hstr - __init_el2_mpam __init_el2_nvhe_idregs __init_el2_cptr __init_el2_fgt diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 96c7aad7cc43ab0cc5c2a828b3f968b48a0b0697..c11010965b8e06e20bffffb8d627817eb5893e75 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -108,7 +108,6 @@ #define HCRX_GUEST_FLAGS (HCRX_EL2_SMPME | HCRX_EL2_TCR2En) #define HCRX_HOST_FLAGS (HCRX_EL2_MSCEn | HCRX_EL2_TCR2En | HCRX_EL2_EnFPM) -#define MPAMHCR_HOST_FLAGS 0 /* TCR_EL2 Registers bits */ #define TCR_EL2_RES1 ((1U << 31) | (1 << 23)) diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h deleted file mode 100644 index 9abe1fe58c34317c16bbc3f4aa6da2fabad74639..0000000000000000000000000000000000000000 --- a/arch/arm64/include/asm/mpam.h +++ /dev/null @@ -1,167 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2021 Arm Ltd. */ - -#ifndef __ASM__MPAM_H -#define __ASM__MPAM_H - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -/* CPU Registers */ -#define MPAM_SYSREG_EN BIT_ULL(63) -#define MPAM_SYSREG_TRAP_IDR BIT_ULL(58) -#define MPAM_SYSREG_TRAP_MPAM0_EL1 BIT_ULL(49) -#define MPAM_SYSREG_TRAP_MPAM1_EL1 BIT_ULL(48) -#define MPAM_SYSREG_PMG_D GENMASK(47, 40) -#define MPAM_SYSREG_PMG_I GENMASK(39, 32) -#define MPAM_SYSREG_PARTID_D GENMASK(31, 16) -#define MPAM_SYSREG_PARTID_I GENMASK(15, 0) - -#define MPAMIDR_PMG_MAX GENMASK(40, 32) -#define MPAMIDR_PMG_MAX_SHIFT 32 -#define MPAMIDR_PMG_MAX_LEN 8 -#define MPAMIDR_VPMR_MAX GENMASK(20, 18) -#define MPAMIDR_VPMR_MAX_SHIFT 18 -#define MPAMIDR_VPMR_MAX_LEN 3 -#define MPAMIDR_HAS_HCR BIT(17) -#define MPAMIDR_HAS_HCR_SHIFT 17 -#define MPAMIDR_PARTID_MAX GENMASK(15, 0) -#define MPAMIDR_PARTID_MAX_SHIFT 0 -#define MPAMIDR_PARTID_MAX_LEN 15 - -#define MPAMHCR_EL0_VPMEN BIT_ULL(0) -#define MPAMHCR_EL1_VPMEN BIT_ULL(1) -#define MPAMHCR_GSTAPP_PLK BIT_ULL(8) -#define MPAMHCR_TRAP_MPAMIDR BIT_ULL(31) - -/* Properties of the VPM registers */ -#define MPAM_VPM_NUM_REGS 8 -#define MPAM_VPM_PARTID_LEN 16 -#define MPAM_VPM_PARTID_MASK 0xffff -#define MPAM_VPM_REG_LEN 64 -#define MPAM_VPM_PARTIDS_PER_REG (MPAM_VPM_REG_LEN / MPAM_VPM_PARTID_LEN) -#define MPAM_VPM_MAX_PARTID (MPAM_VPM_NUM_REGS * MPAM_VPM_PARTIDS_PER_REG) - - -DECLARE_STATIC_KEY_FALSE(arm64_mpam_has_hcr); -DECLARE_STATIC_KEY_FALSE(mpam_enabled); -DECLARE_PER_CPU(u64, arm64_mpam_default); -DECLARE_PER_CPU(u64, arm64_mpam_current); - -/* check whether all CPUs have MPAM support */ -static __always_inline bool mpam_cpus_have_feature(void) -{ - if (IS_ENABLED(CONFIG_ARM64_MPAM)) - return cpus_have_final_cap(ARM64_MPAM); - return false; -} - -/* check whether all CPUs have MPAM virtualisation support */ -static __always_inline bool mpam_cpus_have_mpam_hcr(void) -{ - if (IS_ENABLED(CONFIG_ARM64_MPAM)) - return static_branch_unlikely(&arm64_mpam_has_hcr); - return false; -} - -/* enable MPAM virtualisation support */ -static inline void __init __enable_mpam_hcr(void) -{ - if (IS_ENABLED(CONFIG_ARM64_MPAM)) - static_branch_enable(&arm64_mpam_has_hcr); -} - -/* - * The resctrl filesystem writes to the partid/pmg values for threads and CPUs, - * which may race with reads in __mpam_sched_in(). Ensure only one of the old - * or new values are used. Particular care should be taken with the pmg field - * as __mpam_sched_in() may read a partid and pmg that don't match, causing - * this value to be stored with cache allocations, despite being considered - * 'free' by resctrl. - * - * A value in struct thread_info is used instead of struct task_struct as the - * cpu's u64 register format is used, but struct task_struct has two u32'. - */ - static inline void mpam_set_cpu_defaults(int cpu, u16 partid_d, u16 partid_i, - u8 pmg_d, u8 pmg_i) -{ - u64 default_val; - - default_val = FIELD_PREP(MPAM_SYSREG_PARTID_D, partid_d); - default_val |= FIELD_PREP(MPAM_SYSREG_PARTID_I, partid_i); - default_val |= FIELD_PREP(MPAM_SYSREG_PMG_D, pmg_d); - default_val |= FIELD_PREP(MPAM_SYSREG_PMG_I, pmg_i); - - WRITE_ONCE(per_cpu(arm64_mpam_default, cpu), default_val); -} - -static inline void mpam_set_task_partid_pmg(struct task_struct *tsk, - u16 partid_d, u16 partid_i, - u8 pmg_d, u8 pmg_i) -{ -#ifdef CONFIG_ARM64_MPAM - u64 regval; - - regval = FIELD_PREP(MPAM_SYSREG_PARTID_D, partid_d); - regval |= FIELD_PREP(MPAM_SYSREG_PARTID_I, partid_i); - regval |= FIELD_PREP(MPAM_SYSREG_PMG_D, pmg_d); - regval |= FIELD_PREP(MPAM_SYSREG_PMG_I, pmg_i); - - WRITE_ONCE(task_thread_info(tsk)->mpam_partid_pmg, regval); -#endif -} - -static inline u64 mpam_get_regval(struct task_struct *tsk) -{ -#ifdef CONFIG_ARM64_MPAM - return READ_ONCE(task_thread_info(tsk)->mpam_partid_pmg); -#else - return 0; -#endif -} - -static inline void resctrl_arch_set_rmid(struct task_struct *tsk, u32 rmid) -{ -#ifdef CONFIG_ARM64_MPAM - u64 regval = mpam_get_regval(tsk); - - regval &= ~MPAM_SYSREG_PMG_D; - regval &= ~MPAM_SYSREG_PMG_I; - regval |= FIELD_PREP(MPAM_SYSREG_PMG_D, rmid); - regval |= FIELD_PREP(MPAM_SYSREG_PMG_I, rmid); - - WRITE_ONCE(task_thread_info(tsk)->mpam_partid_pmg, regval); -#endif -} - -static inline void mpam_thread_switch(struct task_struct *tsk) -{ - u64 oldregval; - int cpu = smp_processor_id(); - u64 regval = mpam_get_regval(tsk); - - if (!IS_ENABLED(CONFIG_ARM64_MPAM) || - !static_branch_likely(&mpam_enabled)) - return; - - if (regval == READ_ONCE(mpam_resctrl_default_group)) - regval = READ_ONCE(per_cpu(arm64_mpam_default, cpu)); - - oldregval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); - if (oldregval == regval) - return; - - /* Synchronising this write is left until the ERET to EL0 */ - write_sysreg_s(regval, SYS_MPAM0_EL1); - WRITE_ONCE(per_cpu(arm64_mpam_current, cpu), regval); -} -#endif /* __ASM__MPAM_H */ diff --git a/arch/arm64/include/asm/resctrl.h b/arch/arm64/include/asm/resctrl.h deleted file mode 100644 index b506e95cf6e374690a71e0f1d142fe0032ae6e45..0000000000000000000000000000000000000000 --- a/arch/arm64/include/asm/resctrl.h +++ /dev/null @@ -1,2 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 752e45c29afaf62b36bee74e450e83f4a3b9355c..e4bb2c2a136a8205b8f06050e22614003fc1438d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -503,13 +503,6 @@ #define SYS_MAIR_EL2 sys_reg(3, 4, 10, 2, 0) #define SYS_AMAIR_EL2 sys_reg(3, 4, 10, 3, 0) -#define SYS_MPAMHCR_EL2 sys_reg(3, 4, 10, 4, 0) -#define SYS_MPAMVPMV_EL2 sys_reg(3, 4, 10, 4, 1) -#define SYS_MPAM2_EL2 sys_reg(3, 4, 10, 5, 0) - -#define __VPMn_op2(n) ((n) & 0x7) -#define SYS_MPAM_VPMn_EL2(n) sys_reg(3, 4, 10, 6, __VPMn_op2(n)) - #define SYS_VBAR_EL2 sys_reg(3, 4, 12, 0, 0) #define SYS_RVBAR_EL2 sys_reg(3, 4, 12, 0, 1) #define SYS_RMR_EL2 sys_reg(3, 4, 12, 0, 2) @@ -574,7 +567,6 @@ #define SYS_TFSR_EL12 sys_reg(3, 5, 5, 6, 0) #define SYS_MAIR_EL12 sys_reg(3, 5, 10, 2, 0) #define SYS_AMAIR_EL12 sys_reg(3, 5, 10, 3, 0) -#define SYS_MPAM1_EL12 sys_reg(3, 5, 10, 5, 0) #define SYS_VBAR_EL12 sys_reg(3, 5, 12, 0, 0) #define SYS_CNTKCTL_EL12 sys_reg(3, 5, 14, 1, 0) #define SYS_CNTP_TVAL_EL02 sys_reg(3, 5, 14, 2, 0) diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index f36803eb2ebd277006f46446936b10e63405fdc5..872b8e810a3774f8a8c662d546e5eb0abc7743ab 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -41,9 +41,6 @@ struct thread_info { #ifdef CONFIG_SHADOW_CALL_STACK void *scs_base; void *scs_sp; -#endif -#ifdef CONFIG_ARM64_MPAM - u64 mpam_partid_pmg; #endif u32 cpu; }; diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index e9827519f567097b8cd0e27fb5a6dbd115e2ce92..e3bab76856a391ad83c3b6269b9d9638e0a09829 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -67,13 +67,11 @@ obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \ obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o - obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_CRASH_CORE) += crash_core.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_SDEI_WATCHDOG) += watchdog_sdei.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o -obj-$(CONFIG_ARM64_MPAM) += mpam.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6b50132aed89bc4c77658d3c4bb614f7f384825b..c214639da4a4789f64146ed565c79d6507a1b8ef 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2814,15 +2814,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_fpmr, ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, FPMR, IMP) }, -#ifdef CONFIG_ARM64_MPAM - { - .desc = "Memory Partitioning And Monitoring", - .type = ARM64_CPUCAP_SYSTEM_FEATURE, - .capability = ARM64_MPAM, - .matches = has_cpuid_feature, - ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1) - }, -#endif #ifdef CONFIG_ARM64_POE { .desc = "Stage-1 Permission Overlay Extension (S1POE)", diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 6999668e9ecf0811258a02acefabde99543bae49..35f3c795951373549faad3ed2d1bd30ad427eb78 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -64,12 +64,6 @@ KVM_NVHE_ALIAS(nvhe_hyp_panic_handler); /* Vectors installed by hyp-init on reset HVC. */ KVM_NVHE_ALIAS(__hyp_stub_vectors); -/* Additional static keys for cpufeatures */ -#ifdef CONFIG_ARM64_MPAM -KVM_NVHE_ALIAS(arm64_mpam_has_hcr); -KVM_NVHE_ALIAS(mpam_enabled); -#endif - /* Static keys which are set if a vGIC trap should be handled in hyp. */ KVM_NVHE_ALIAS(vgic_v2_cpuif_trap); KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); diff --git a/arch/arm64/kernel/mpam.c b/arch/arm64/kernel/mpam.c deleted file mode 100644 index 190a0aa4f9c0050f2f1e6fad5e5d1fccbbcd62e0..0000000000000000000000000000000000000000 --- a/arch/arm64/kernel/mpam.c +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2021 Arm Ltd. */ - -#include - -#include -#include -#include - -DEFINE_STATIC_KEY_FALSE(arm64_mpam_has_hcr); -EXPORT_SYMBOL_FOR_KVM(arm64_mpam_has_hcr); -DEFINE_STATIC_KEY_FALSE(mpam_enabled); -EXPORT_SYMBOL_FOR_KVM(mpam_enabled); -DEFINE_PER_CPU(u64, arm64_mpam_default); -DEFINE_PER_CPU(u64, arm64_mpam_current); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index c3b83023851f5ac6d02fd05505a2e9c058de75b9..a44f0e97f6d7cd766185d65fdc602e87bd403ee1 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -49,7 +49,6 @@ #include #include #include -#include #include #include #include @@ -578,12 +577,6 @@ struct task_struct *__switch_to(struct task_struct *prev, if (prev->thread.sctlr_user != next->thread.sctlr_user) update_sctlr_el1(next->thread.sctlr_user); - /* - * MPAM thread switch happens after the DSB to ensure prev's accesses - * use prev's MPAM settings. - */ - mpam_thread_switch(next); - /* the actual thread switch */ last = cpu_switch_to(prev, next); diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index d274244e1b44097758922e47dc025db1ebbe073c..2b6c9724d55fdd3ee99b21f9234cd0c1fcbfe941 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -173,35 +172,6 @@ static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) write_sysreg_s(ctxt_sys_reg(hctxt, HDFGWTR_EL2), SYS_HDFGWTR_EL2); } -static inline void __activate_traps_mpam(struct kvm_vcpu *vcpu) -{ - u64 r = MPAM_SYSREG_TRAP_MPAM0_EL1 | MPAM_SYSREG_TRAP_MPAM1_EL1; - - if (!mpam_cpus_have_feature() || !static_branch_likely(&mpam_enabled)) - return; - - /* trap guest access to MPAMIDR_EL1 */ - if (mpam_cpus_have_mpam_hcr()) { - write_sysreg_s(MPAMHCR_TRAP_MPAMIDR, SYS_MPAMHCR_EL2); - } else { - /* From v1.1 TIDR can trap MPAMIDR, set it unconditionally */ - r |= MPAM_SYSREG_TRAP_IDR; - } - - write_sysreg_s(r, SYS_MPAM2_EL2); -} - -static inline void __deactivate_traps_mpam(void) -{ - if (!mpam_cpus_have_feature() || !static_branch_likely(&mpam_enabled)) - return; - - write_sysreg_s(0, SYS_MPAM2_EL2); - - if (mpam_cpus_have_mpam_hcr()) - write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2); -} - static inline void __activate_traps_common(struct kvm_vcpu *vcpu) { /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */ @@ -242,7 +212,6 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) } __activate_traps_hfgxtr(vcpu); - __activate_traps_mpam(vcpu); } static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) @@ -262,7 +231,6 @@ static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) write_sysreg_s(HCRX_HOST_FLAGS, SYS_HCRX_EL2); __deactivate_traps_hfgxtr(vcpu); - __deactivate_traps_mpam(); } static inline void ___activate_traps(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index 8e99f66b377bd37f3622ae9b5c9bf881517659b8..bb6b571ec627dede466c5fa1d05785a9c9f78764 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -15,7 +15,6 @@ #include #include #include -#include static inline void __sysreg_save_common_state(struct kvm_cpu_context *ctxt) { @@ -244,32 +243,4 @@ static inline void __sysreg32_restore_state(struct kvm_vcpu *vcpu) write_sysreg(__vcpu_sys_reg(vcpu, DBGVCR32_EL2), dbgvcr32_el2); } -/* - * The _EL0 value was written by the host's context switch, copy this into the - * guest's EL1. - */ -static inline void __mpam_guest_load(void) -{ - if (IS_ENABLED(CONFIG_ARM64_MPAM) && mpam_cpus_have_feature() && - static_branch_likely(&mpam_enabled)) - write_sysreg_el1(read_sysreg_s(SYS_MPAM0_EL1), SYS_MPAM1); -} - -/* - * Copy the _EL2 register back to _EL1, clearing any trap bits EL2 may have set. - * nVHE world-switch copies the _EL1 register to _EL2. A VHE host writes to the - * _EL2 register as it is aliased by the hardware when TGE is set. - */ -static inline void __mpam_guest_put(void) -{ - u64 val, mask = MPAM_SYSREG_PMG_D | MPAM_SYSREG_PMG_I | - MPAM_SYSREG_PARTID_D | MPAM_SYSREG_PARTID_I; - - if (IS_ENABLED(CONFIG_ARM64_MPAM) && mpam_cpus_have_feature() && - static_branch_likely(&mpam_enabled)) { - val = FIELD_GET(mask, read_sysreg_s(SYS_MPAM2_EL2)); - write_sysreg_el1(val, SYS_MPAM1); - } -} - #endif /* __ARM64_KVM_HYP_SYSREG_SR_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 024a7871892f4ff81f0447c535e40eae46daf6d2..f1970816fa436d8f2b1d67517840de4bdd73b9c6 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -282,14 +282,6 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) return __fixup_guest_exit(vcpu, exit_code, handlers); } -/* Use the host thread's partid and pmg for world switch */ -static void __mpam_copy_el1_to_el2(void) -{ - if (IS_ENABLED(CONFIG_ARM64_MPAM) && mpam_cpus_have_feature() && - static_branch_likely(&mpam_enabled)) - write_sysreg_s(read_sysreg_s(SYS_MPAM1_EL1), SYS_MPAM2_EL2); -} - /* Switch to the guest for legacy non-VHE systems */ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) { @@ -299,8 +291,6 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) bool pmu_switch_needed; u64 exit_code; - __mpam_copy_el1_to_el2(); - /* * Having IRQs masked via PMR when entering the guest means the GIC * will not signal the CPU of interrupts of lower priority, and the @@ -360,7 +350,6 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) __timer_enable_traps(vcpu); __debug_switch_to_guest(vcpu); - __mpam_guest_load(); do { /* Jump in the fire! */ @@ -371,7 +360,6 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) __sysreg_save_state_nvhe(guest_ctxt); __sysreg32_save_state(vcpu); - __mpam_guest_put(); __timer_disable_traps(vcpu); __hyp_vgic_save_state(vcpu); diff --git a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c index 71af2f8c7568eeb68e603ba2e00a32cfdb0ee16f..ae763061909ac4cf3619f11ce9d2f7b289f330b2 100644 --- a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c @@ -90,7 +90,6 @@ void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu) __sysreg32_restore_state(vcpu); __sysreg_restore_user_state(guest_ctxt); __sysreg_restore_el1_state(guest_ctxt); - __mpam_guest_load(); vcpu_set_flag(vcpu, SYSREGS_ON_CPU); } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7738226ef8cf12e56e7577d6aa386bb5c620f531..623737f65228389b3088a667eb7e738c80aa50ad 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -418,31 +418,6 @@ static bool trap_oslar_el1(struct kvm_vcpu *vcpu, return true; } -static bool trap_mpam(struct kvm_vcpu *vcpu, - struct sys_reg_params *p, - const struct sys_reg_desc *r) -{ - u64 aa64pfr0_el1 = IDREG(vcpu->kvm, SYS_ID_AA64PFR0_EL1); - - /* - * What did we expose to the guest? - * Earlier guests may have seen the ID bits, which can't be removed - * without breaking migration, but MPAMIDR_EL1 can advertise all-zeroes, - * indicating there are zero PARTID/PMG supported by the CPU, allowing - * the other two trapped registers (MPAM1_EL1 and MPAM0_EL1) to be - * treated as RAZ/WI. - * Emulating MPAM1_EL1 as RAZ/WI means the guest sees the MPAMEN bit - * as clear, and realises MPAM isn't usable on this CPU. - */ - if (FIELD_GET(ID_AA64PFR0_EL1_MPAM_MASK, aa64pfr0_el1)) { - p->regval = 0; - return true; - } - - kvm_inject_undefined(vcpu); - return false; -} - static bool trap_oslsr_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) @@ -1341,36 +1316,6 @@ static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp, return arm64_ftr_safe_value(&kvm_ftr, new, cur); } -static u64 kvm_arm64_ftr_max(struct kvm_vcpu *vcpu, - const struct sys_reg_desc *rd) -{ - u64 pfr0, val = rd->reset(vcpu, rd); - u32 field, id = reg_to_encoding(rd); - - /* - * Some values may reset to a lower value than can be supported, - * get the maximum feature value. - */ - switch (id) { - case SYS_ID_AA64PFR0_EL1: - pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); - - /* - * MPAM resets to 0, but migration of MPAM=1 guests is needed. - * See trap_mpam() for more. - */ - field = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT); - if (field == ID_AA64PFR0_EL1_MPAM_1) { - val &= ~ID_AA64PFR0_EL1_MPAM_MASK; - val |= FIELD_PREP(ID_AA64PFR0_EL1_MPAM_MASK, ID_AA64PFR0_EL1_MPAM_1); - } - - break; - } - - return val; -} - /** * arm64_check_features() - Check if a feature register value constitutes * a subset of features indicated by the idreg's KVM sanitised limit. @@ -1391,7 +1336,8 @@ static int arm64_check_features(struct kvm_vcpu *vcpu, const struct arm64_ftr_bits *ftrp = NULL; u32 id = reg_to_encoding(rd); u64 writable_mask = rd->val; - u64 limit, mask = 0; + u64 limit = rd->reset(vcpu, rd); + u64 mask = 0; u32 midr = read_cpuid_id(); /* @@ -1406,7 +1352,6 @@ static int arm64_check_features(struct kvm_vcpu *vcpu, if (!ftr_reg) return -EINVAL; - limit = kvm_arm64_ftr_max(vcpu, rd); ftrp = ftr_reg->ftr_bits; for (; ftrp && ftrp->width; ftrp++) { @@ -1624,8 +1569,7 @@ static u64 read_sanitised_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, /* * MPAM is disabled by default as KVM also needs a set of PARTID to * program the MPAMVPMx_EL2 PARTID remapping registers with. But some - * older kernels let the guest see the ID bit. Turning it on causes - * the registers to be emulated as RAZ/WI. See trap_mpam() for more. + * older kernels let the guest see the ID bit. */ val &= ~ID_AA64PFR0_EL1_MPAM_MASK; @@ -2393,11 +2337,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_LOREA_EL1), trap_loregion }, { SYS_DESC(SYS_LORN_EL1), trap_loregion }, { SYS_DESC(SYS_LORC_EL1), trap_loregion }, - { SYS_DESC(SYS_MPAMIDR_EL1), trap_mpam }, { SYS_DESC(SYS_LORID_EL1), trap_loregion }, - { SYS_DESC(SYS_MPAM1_EL1), trap_mpam }, - { SYS_DESC(SYS_MPAM0_EL1), trap_mpam }, { SYS_DESC(SYS_VBAR_EL1), access_rw, reset_val, VBAR_EL1, 0 }, { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 }, diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9946e38b2b6587017f332dcda17ab6753de4678f..0f68f9430e44f068a7bc0b1d4243fcac330dbdb8 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -59,7 +59,6 @@ HW_DBM KVM_HVHE KVM_PROTECTED_MODE MISMATCHED_CACHE_TYPE -MPAM MTE MTE_ASYMM SME diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 4ff32f97fcc1cb0cc98bd84cb3bb8a61399df5c5..d32d591859bf25640a613313e9267390674fa866 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3182,22 +3182,6 @@ Res0 1 Field 0 EN EndSysreg -Sysreg MPAMIDR_EL1 3 0 10 4 4 -Res0 63:62 -Field 61 HAS_SDEFLT -Field 60 HAS_FORCE_NS -Field 59 SP4 -Field 58 HAS_TIDR -Field 57 HAS_ALTSP -Res0 56:40 -Field 39:32 PMG_MAX -Res0 31:21 -Field 20:18 VPMR_MAX -Field 17 HAS_HCR -Res0 16 -Field 15:0 PARTID_MAX -EndSysreg - Sysreg LORID_EL1 3 0 10 4 7 Res0 63:24 Field 23:16 LD @@ -3205,22 +3189,6 @@ Res0 15:8 Field 7:0 LR EndSysreg -Sysreg MPAM1_EL1 3 0 10 5 0 -Res0 63:48 -Field 47:40 PMG_D -Field 39:32 PMG_I -Field 31:16 PARTID_D -Field 15:0 PARTID_I -EndSysreg - -Sysreg MPAM0_EL1 3 0 10 5 1 -Res0 63:48 -Field 47:40 PMG_D -Field 39:32 PMG_I -Field 31:16 PARTID_D -Field 15:0 PARTID_I -EndSysreg - Sysreg ISR_EL1 3 0 12 1 0 Res0 63:11 Field 10 IS @@ -3234,7 +3202,6 @@ EndSysreg Sysreg ICC_NMIAR1_EL1 3 0 12 9 5 Res0 63:24 Field 23:0 INTID - EndSysreg Sysreg TRBLIMITR_EL1 3 0 9 11 0 diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index cc84cef105cfc0944e7162ec4e148aee3760b1cf..82dafbe6af59b330eb3da9e341eec133ef18fa6c 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -493,7 +493,6 @@ config X86_CPU_RESCTRL bool "x86 CPU resource control support" depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD) depends on MISC_FILESYSTEMS - select KERNFS select ARCH_HAS_CPU_RESCTRL select RESCTRL_FS select RESCTRL_FS_PSEUDO_LOCK diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 5172ff7e623fb83b67038e62e7e4196b63d0a13f..fdac919c8cc568a8c7b2dbca34ba723cbddcbe01 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -511,12 +511,9 @@ #define X86_FEATURE_TSA_SQ_NO (21*32+11) /* "" AMD CPU not vulnerable to TSA-SQ */ #define X86_FEATURE_TSA_L1_NO (21*32+12) /* "" AMD CPU not vulnerable to TSA-L1 */ #define X86_FEATURE_CLEAR_CPU_BUF_VM (21*32+13) /* "" Clear CPU buffers using VERW before VMRUN */ -#define X86_FEATURE_ABMC (21*32+ 6) /* "" Assignable Bandwidth Monitoring Counters */ #define X86_FEATURE_AMD_FAST_CPPC (21*32 + 5) /* Fast CPPC */ #define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32 + 7) /* Workload Classification */ -#define X86_FEATURE_SDCIAE (21*32+18) /* L3 Smart Data Cache Injection Allocation Enforcement */ - /* * BUG word(s) */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 39d111515bd03539ba541b394e32922a1a6a36d8..5afb07c04669398752d636050e00bdcadf8ab08a 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -1187,6 +1187,7 @@ #define MSR_IA32_QM_CTR 0xc8e #define MSR_IA32_PQR_ASSOC 0xc8f #define MSR_IA32_L3_CBM_BASE 0xc90 +#define MSR_RMID_SNC_CONFIG 0xca0 #define MSR_IA32_L2_CBM_BASE 0xd10 #define MSR_IA32_MBA_THRTL_BASE 0xd50 @@ -1194,9 +1195,6 @@ #define MSR_IA32_MBA_BW_BASE 0xc0000200 #define MSR_IA32_SMBA_BW_BASE 0xc0000280 #define MSR_IA32_EVT_CFG_BASE 0xc0000400 -#define MSR_IA32_L3_QOS_EXT_CFG 0xc00003ff -#define MSR_IA32_L3_QOS_ABMC_CFG 0xc00003fd -#define MSR_IA32_L3_QOS_ABMC_DSC 0xc00003fe /* MSR_IA32_VMX_MISC bits */ #define MSR_IA32_VMX_MISC_INTEL_PT (1ULL << 14) diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h index 7ea7c3d7b5bebd31a912495649ef0768b895b6b0..ad497ab196d1e755b598b6fc4f365fa49d79fb10 100644 --- a/arch/x86/include/asm/resctrl.h +++ b/arch/x86/include/asm/resctrl.h @@ -6,8 +6,8 @@ #include #include -#include #include +#include /* * This value can never be a valid CLOSID, and is used when mapping a @@ -16,8 +16,6 @@ */ #define X86_RESCTRL_EMPTY_CLOSID ((u32)~0) -void resctrl_arch_reset_resources(void); - /** * struct resctrl_pqr_state - State cache for the PQR MSR * @cur_rmid: The cached Resource Monitoring ID @@ -99,11 +97,6 @@ static inline bool resctrl_arch_is_mbm_local_enabled(void) return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID)); } -static inline bool resctrl_arch_is_mbm_bps_enabled(void) -{ - return false; -} - /* * __resctrl_sched_in() - Writes the task's CLOSid/RMID to IA32_PQR_MSR * @@ -188,12 +181,6 @@ static inline void resctrl_arch_sched_in(struct task_struct *tsk) __resctrl_sched_in(tsk); } -static inline u32 resctrl_arch_system_num_rmid_idx(void) -{ - /* RMID are independent numbers for x86. num_rmid_idx == num_rmid */ - return boot_cpu_data.x86_cache_max_rmid + 1; -} - static inline void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid) { *rmid = idx; @@ -207,41 +194,19 @@ static inline u32 resctrl_arch_rmid_idx_encode(u32 ignored, u32 rmid) /* x86 can always read an rmid, nothing needs allocating */ struct rdt_resource; -static inline void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, int evtid) +static inline void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, + enum resctrl_event_id evtid) { might_sleep(); return NULL; -}; - -static inline void resctrl_arch_mon_ctx_free(struct rdt_resource *r, int evtid, - void *ctx) { }; - -int resctrl_arch_mbm_cntr_assign_enable(void); -void resctrl_arch_mbm_cntr_assign_configure(void); -void resctrl_arch_mbm_cntr_assign_disable(void); -bool resctrl_arch_get_mbm_cntr_assign_enable(void); - -void resctrl_arch_event_config_set(void *info); -u32 resctrl_arch_event_config_get(void *d, enum resctrl_event_id eventid); +} -int resctrl_arch_assign_cntr(void *dom, enum resctrl_event_id evtid, - u32 rmid, u32 cntr_id, u32 closid, bool assign); +static inline void resctrl_arch_mon_ctx_free(struct rdt_resource *r, + enum resctrl_event_id evtid, + void *ctx) { } -u64 resctrl_arch_get_prefetch_disable_bits(void); -int resctrl_arch_pseudo_lock_fn(void *_plr); -int resctrl_arch_measure_cycles_lat_fn(void *_plr); -int resctrl_arch_measure_l2_residency(void *_plr); -int resctrl_arch_measure_l3_residency(void *_plr); void resctrl_cpu_detect(struct cpuinfo_x86 *c); -bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level l); -int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable); - -bool resctrl_arch_is_hwdrc_mb_capable(void); -int resctrl_arch_set_hwdrc_enabled(enum resctrl_res_level l, bool hwdrc_mb); - -bool resctrl_arch_get_abmc_enabled(void); - #else static inline void resctrl_arch_sched_in(struct task_struct *tsk) {} diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c index 8be2847473c4ce0e07bc17b7b61528ebcf4f4dbd..7d8733874218d0fd6e0853a216659446ad96803d 100644 --- a/arch/x86/kernel/cpu/cpuid-deps.c +++ b/arch/x86/kernel/cpu/cpuid-deps.c @@ -70,10 +70,6 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_CQM_MBM_LOCAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_BMEC, X86_FEATURE_CQM_MBM_TOTAL }, { X86_FEATURE_BMEC, X86_FEATURE_CQM_MBM_LOCAL }, - { X86_FEATURE_ABMC, X86_FEATURE_CQM_MBM_TOTAL }, - { X86_FEATURE_ABMC, X86_FEATURE_CQM_MBM_LOCAL }, - { X86_FEATURE_ABMC, X86_FEATURE_BMEC }, - { X86_FEATURE_SDCIAE, X86_FEATURE_CAT_L3 }, { X86_FEATURE_AVX512_BF16, X86_FEATURE_AVX512VL }, { X86_FEATURE_AVX512_FP16, X86_FEATURE_AVX512BW }, { X86_FEATURE_ENQCMD, X86_FEATURE_XSAVES }, diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 6c5a17d7a4a3554600fe6975d537750017d78ebd..1491a111e619610739a69671436bdcbeab48c424 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -18,7 +18,6 @@ #include #include #include -#include #include "cpu.h" @@ -121,7 +120,6 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c) x86_amd_ls_cfg_ssbd_mask = 1ULL << 10; } } - resctrl_cpu_detect(c); } static void init_hygon_cap(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/resctrl/Makefile b/arch/x86/kernel/cpu/resctrl/Makefile index 0c13b0befd8a9b76fe4753b8ee230113e5f3bf54..d8a04b195da212990289aa95d50fe681a0eea44c 100644 --- a/arch/x86/kernel/cpu/resctrl/Makefile +++ b/arch/x86/kernel/cpu/resctrl/Makefile @@ -2,4 +2,6 @@ obj-$(CONFIG_X86_CPU_RESCTRL) += core.o rdtgroup.o monitor.o obj-$(CONFIG_X86_CPU_RESCTRL) += ctrlmondata.o obj-$(CONFIG_RESCTRL_FS_PSEUDO_LOCK) += pseudo_lock.o + +# To allow define_trace.h's recursive include: CFLAGS_pseudo_lock.o = -I$(src) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 9f1a439be5abe1a7e0a95dafc2f5ee37ee2e2602..548390bfde3a8f94a387c6c0fbde94570a0af3e0 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -19,10 +19,9 @@ #include #include #include -#include #include -#include +#include #include #include "internal.h" @@ -50,27 +49,23 @@ DEFINE_PER_CPU(struct resctrl_pqr_state, pqr_state); */ bool rdt_alloc_capable; -static void -mba_wrmsr_intel(struct rdt_domain *d, struct msr_param *m, - struct rdt_resource *r); -static void -cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r); -static void -mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m, - struct rdt_resource *r); +static void mba_wrmsr_intel(struct msr_param *m); +static void cat_wrmsr(struct msr_param *m); +static void mba_wrmsr_amd(struct msr_param *m); -#define domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.domains) +#define ctrl_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.ctrl_domains) +#define mon_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.mon_domains) -struct rdt_hw_resource rdt_resources_all[] = { +struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = { [RDT_RESOURCE_L3] = { .r_resctrl = { - .rid = RDT_RESOURCE_L3, .name = "L3", - .cache_level = 3, - .domains = domain_init(RDT_RESOURCE_L3), - .format_str = "%d=%0*x", - .fflags = RFTYPE_RES_CACHE, + .ctrl_scope = RESCTRL_L3_CACHE, + .mon_scope = RESCTRL_L3_CACHE, + .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L3), + .mon_domains = mon_domain_init(RDT_RESOURCE_L3), + .schema_fmt = RESCTRL_SCHEMA_BITMAP, }, .msr_base = MSR_IA32_L3_CBM_BASE, .msr_update = cat_wrmsr, @@ -78,12 +73,10 @@ struct rdt_hw_resource rdt_resources_all[] = { [RDT_RESOURCE_L2] = { .r_resctrl = { - .rid = RDT_RESOURCE_L2, .name = "L2", - .cache_level = 2, - .domains = domain_init(RDT_RESOURCE_L2), - .format_str = "%d=%0*x", - .fflags = RFTYPE_RES_CACHE, + .ctrl_scope = RESCTRL_L2_CACHE, + .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_L2), + .schema_fmt = RESCTRL_SCHEMA_BITMAP, }, .msr_base = MSR_IA32_L2_CBM_BASE, .msr_update = cat_wrmsr, @@ -91,27 +84,31 @@ struct rdt_hw_resource rdt_resources_all[] = { [RDT_RESOURCE_MBA] = { .r_resctrl = { - .rid = RDT_RESOURCE_MBA, .name = "MB", - .cache_level = 3, - .domains = domain_init(RDT_RESOURCE_MBA), - .format_str = "%d=%*u", - .fflags = RFTYPE_RES_MB, + .ctrl_scope = RESCTRL_L3_CACHE, + .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_MBA), + .schema_fmt = RESCTRL_SCHEMA_RANGE, }, }, [RDT_RESOURCE_SMBA] = { .r_resctrl = { - .rid = RDT_RESOURCE_SMBA, .name = "SMBA", - .cache_level = 3, - .domains = domain_init(RDT_RESOURCE_SMBA), - .format_str = "%d=%*u", - .fflags = RFTYPE_RES_MB, + .ctrl_scope = RESCTRL_L3_CACHE, + .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_SMBA), + .schema_fmt = RESCTRL_SCHEMA_RANGE, }, }, }; +u32 resctrl_arch_system_num_rmid_idx(void) +{ + struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; + + /* RMID are independent numbers for x86. num_rmid_idx == num_rmid */ + return r->num_rmid; +} + struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) { if (l >= RDT_NUM_RESOURCES) @@ -154,7 +151,6 @@ static inline void cache_alloc_hsw_probe(void) return; hw_res->num_closid = 4; - r->default_ctrl = max_cbm; r->cache.cbm_len = 20; r->cache.shareable_bits = 0xc0000; r->cache.min_cbm_bits = 2; @@ -195,7 +191,7 @@ static __init bool __get_mem_config_intel(struct rdt_resource *r) cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full); hw_res->num_closid = edx.split.cos_max + 1; max_delay = eax.split.max_delay + 1; - r->default_ctrl = MAX_MBA_BW; + r->membw.max_bw = MAX_MBA_BW; r->membw.arch_needs_linear = true; if (ecx & MBA_IS_LINEAR) { r->membw.delay_linear = true; @@ -206,7 +202,6 @@ static __init bool __get_mem_config_intel(struct rdt_resource *r) return false; r->membw.arch_needs_linear = false; } - r->data_width = 3; if (boot_cpu_has(X86_FEATURE_PER_THREAD_MBA)) r->membw.throttle_mode = THREAD_THROTTLE_PER_THREAD; @@ -231,7 +226,7 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx); hw_res->num_closid = edx + 1; - r->default_ctrl = 1 << eax; + r->membw.max_bw = 1 << eax; /* AMD does not use delay */ r->membw.delay_linear = false; @@ -244,8 +239,6 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; r->membw.min_bw = 0; r->membw.bw_gran = 1; - /* Max value is 2048, Data width should be 4 in decimal */ - r->data_width = 4; r->alloc_capable = true; @@ -258,14 +251,13 @@ static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r) union cpuid_0x10_1_eax eax; union cpuid_0x10_x_ecx ecx; union cpuid_0x10_x_edx edx; - u32 ebx; + u32 ebx, default_ctrl; cpuid_count(0x00000010, idx, &eax.full, &ebx, &ecx.full, &edx.full); hw_res->num_closid = edx.split.cos_max + 1; r->cache.cbm_len = eax.split.cbm_len + 1; - r->default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1; - r->cache.shareable_bits = ebx & r->default_ctrl; - r->data_width = (r->cache.cbm_len + 3) / 4; + default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1; + r->cache.shareable_bits = ebx & default_ctrl; if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) r->cache.arch_has_sparse_bitmasks = ecx.split.noncont; r->alloc_capable = true; @@ -281,11 +273,6 @@ static void rdt_get_cdp_config(int level) rdt_resources_all[level].r_resctrl.cdp_capable = true; } -static void rdt_set_io_alloc_capable(struct rdt_resource *r) -{ - r->cache.io_alloc_capable = true; -} - static void rdt_get_cdp_l3_config(void) { rdt_get_cdp_config(RDT_RESOURCE_L3); @@ -296,17 +283,11 @@ static void rdt_get_cdp_l2_config(void) rdt_get_cdp_config(RDT_RESOURCE_L2); } -bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level l) -{ - return rdt_resources_all[l].cdp_enabled; -} - -static void -mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r) +static void mba_wrmsr_amd(struct msr_param *m) { + struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); + struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); unsigned int i; - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); for (i = m->low; i < m->high; i++) wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]); @@ -323,28 +304,25 @@ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) return MAX_MBA_BW - bw; pr_warn_once("Non Linear delay-bw map not supported but queried\n"); - return r->default_ctrl; + return MAX_MBA_BW; } -static void -mba_wrmsr_intel(struct rdt_domain *d, struct msr_param *m, - struct rdt_resource *r) +static void mba_wrmsr_intel(struct msr_param *m) { + struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); + struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); unsigned int i; - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); /* Write the delay values for mba. */ for (i = m->low; i < m->high; i++) - wrmsrl(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], r)); + wrmsrl(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res)); } -static void -cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r) +static void cat_wrmsr(struct msr_param *m) { + struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom); + struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); unsigned int i; - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); for (i = m->low; i < m->high; i++) wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]); @@ -357,57 +335,11 @@ u32 resctrl_arch_get_num_closid(struct rdt_resource *r) void rdt_ctrl_update(void *arg) { + struct rdt_hw_resource *hw_res; struct msr_param *m = arg; - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res); - struct rdt_resource *r = m->res; - int cpu = smp_processor_id(); - struct rdt_domain *d; - d = resctrl_get_domain_from_cpu(cpu, r); - if (d) { - hw_res->msr_update(d, m, r); - return; - } - pr_warn_once("cpu %d not found in any domain for resource %s\n", - cpu, r->name); -} - -/* - * rdt_find_domain - Find a domain in a resource that matches input resource id - * - * Search resource r's domain list to find the resource id. If the resource - * id is found in a domain, return the domain. Otherwise, if requested by - * caller, return the first domain whose id is bigger than the input id. - * The domain list is sorted by id in ascending order. - */ -static struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id, - struct list_head **pos) -{ - struct rdt_domain *d; - struct list_head *l; - - if (id < 0) - return ERR_PTR(-ENODEV); - - list_for_each(l, &r->domains) { - d = list_entry(l, struct rdt_domain, list); - /* When id is found, return its domain. */ - if (id == d->id) - return d; - /* Stop searching when finding id's position in sorted list. */ - if (id < d->id) - break; - } - - if (pos) - *pos = l; - - return NULL; -} - -struct rdt_domain *resctrl_arch_find_domain(struct rdt_resource *r, int id) -{ - return rdt_find_domain(r, id, NULL); + hw_res = resctrl_to_arch_res(m->res); + hw_res->msr_update(m); } static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc) @@ -421,21 +353,26 @@ static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc) * For Memory Allocation: Set b/w requested to 100% */ for (i = 0; i < hw_res->num_closid; i++, dc++) - *dc = r->default_ctrl; + *dc = resctrl_get_default_ctrl(r); +} + +static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom) +{ + kfree(hw_dom->ctrl_val); + kfree(hw_dom); } -static void domain_free(struct rdt_hw_domain *hw_dom) +static void mon_domain_free(struct rdt_hw_mon_domain *hw_dom) { kfree(hw_dom->arch_mbm_total); kfree(hw_dom->arch_mbm_local); - kfree(hw_dom->ctrl_val); kfree(hw_dom); } -static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) +static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_ctrl_domain *d) { + struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d); struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); struct msr_param m; u32 *dc; @@ -447,9 +384,11 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) hw_dom->ctrl_val = dc; setup_default_ctrlval(r, dc); + m.res = r; + m.dom = d; m.low = 0; m.high = hw_res->num_closid; - hw_res->msr_update(d, &m, r); + hw_res->msr_update(&m); return 0; } @@ -458,7 +397,7 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) * @num_rmid: The size of the MBM counter array * @hw_dom: The domain that owns the allocated arrays */ -static int arch_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_domain *hw_dom) +static int arch_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_mon_domain *hw_dom) { size_t tsize; @@ -481,41 +420,47 @@ static int arch_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_domain *hw_dom) return 0; } -/* - * domain_add_cpu - Add a cpu to a resource's domain list. - * - * If an existing domain in the resource r's domain list matches the cpu's - * resource id, add the cpu in the domain. - * - * Otherwise, a new domain is allocated and inserted into the right position - * in the domain list sorted by id in ascending order. - * - * The order in the domain list is visible to users when we print entries - * in the schemata file and schemata input is validated to have the same order - * as this list. - */ -static void domain_add_cpu(int cpu, struct rdt_resource *r) +static int get_domain_id_from_scope(int cpu, enum resctrl_scope scope) +{ + switch (scope) { + case RESCTRL_L2_CACHE: + case RESCTRL_L3_CACHE: + return get_cpu_cacheinfo_id(cpu, scope); + case RESCTRL_L3_NODE: + return cpu_to_node(cpu); + default: + break; + } + + return -EINVAL; +} + +static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r) { - int id = get_cpu_cacheinfo_id(cpu, r->cache_level); + int id = get_domain_id_from_scope(cpu, r->ctrl_scope); + struct rdt_hw_ctrl_domain *hw_dom; struct list_head *add_pos = NULL; - struct rdt_hw_domain *hw_dom; - struct rdt_domain *d; + struct rdt_domain_hdr *hdr; + struct rdt_ctrl_domain *d; int err; - BUG_ON(id > NR_CPUS); lockdep_assert_held(&domain_list_lock); - d = rdt_find_domain(r, id, &add_pos); - if (IS_ERR(d)) { - pr_warn("Couldn't find cache id for CPU %d\n", cpu); + if (id < 0) { + pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n", + cpu, r->ctrl_scope, r->name); return; } - if (d) { - cpumask_set_cpu(cpu, &d->cpu_mask); + hdr = resctrl_find_domain(&r->ctrl_domains, id, &add_pos); + if (hdr) { + if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN)) + return; + d = container_of(hdr, struct rdt_ctrl_domain, hdr); + + cpumask_set_cpu(cpu, &d->hdr.cpu_mask); if (r->cache.arch_has_per_cpu_cfg) rdt_domain_reconfigure_cdp(r); - resctrl_arch_mbm_cntr_assign_configure(); return; } @@ -524,71 +469,189 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r) return; d = &hw_dom->d_resctrl; - d->id = id; - r->rdt_domain_list[id] = d; - - cpumask_set_cpu(cpu, &d->cpu_mask); + d->hdr.id = id; + d->hdr.type = RESCTRL_CTRL_DOMAIN; + cpumask_set_cpu(cpu, &d->hdr.cpu_mask); rdt_domain_reconfigure_cdp(r); - if (r->alloc_capable && domain_setup_ctrlval(r, d)) { - domain_free(hw_dom); + if (domain_setup_ctrlval(r, d)) { + ctrl_domain_free(hw_dom); return; } - resctrl_mbm_evt_config_init(hw_dom); - resctrl_arch_mbm_cntr_assign_configure(); + list_add_tail_rcu(&d->hdr.list, add_pos); - if (r->mon_capable && arch_domain_mbm_alloc(r->mon.num_rmid, hw_dom)) { - domain_free(hw_dom); + err = resctrl_online_ctrl_domain(r, d); + if (err) { + list_del_rcu(&d->hdr.list); + synchronize_rcu(); + ctrl_domain_free(hw_dom); + } +} + +static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) +{ + int id = get_domain_id_from_scope(cpu, r->mon_scope); + struct list_head *add_pos = NULL; + struct rdt_hw_mon_domain *hw_dom; + struct rdt_domain_hdr *hdr; + struct rdt_mon_domain *d; + struct cacheinfo *ci; + int err; + + lockdep_assert_held(&domain_list_lock); + + if (id < 0) { + pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n", + cpu, r->mon_scope, r->name); + return; + } + + hdr = resctrl_find_domain(&r->mon_domains, id, &add_pos); + if (hdr) { + if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN)) + return; + d = container_of(hdr, struct rdt_mon_domain, hdr); + + cpumask_set_cpu(cpu, &d->hdr.cpu_mask); + return; + } + + hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu)); + if (!hw_dom) + return; + + d = &hw_dom->d_resctrl; + d->hdr.id = id; + d->hdr.type = RESCTRL_MON_DOMAIN; + ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE); + if (!ci) { + pr_warn_once("Can't find L3 cache for CPU:%d resource %s\n", cpu, r->name); + mon_domain_free(hw_dom); + return; + } + d->ci_id = ci->id; + cpumask_set_cpu(cpu, &d->hdr.cpu_mask); + + arch_mon_domain_online(r, d); + + if (arch_domain_mbm_alloc(r->num_rmid, hw_dom)) { + mon_domain_free(hw_dom); return; } - list_add_tail_rcu(&d->list, add_pos); + list_add_tail_rcu(&d->hdr.list, add_pos); - err = resctrl_online_domain(r, d); + err = resctrl_online_mon_domain(r, d); if (err) { - list_del_rcu(&d->list); + list_del_rcu(&d->hdr.list); synchronize_rcu(); - domain_free(hw_dom); + mon_domain_free(hw_dom); } } -static void domain_remove_cpu(int cpu, struct rdt_resource *r) +static void domain_add_cpu(int cpu, struct rdt_resource *r) { - int id = get_cpu_cacheinfo_id(cpu, r->cache_level); - struct rdt_hw_domain *hw_dom; - struct rdt_domain *d; + if (r->alloc_capable) + domain_add_cpu_ctrl(cpu, r); + if (r->mon_capable) + domain_add_cpu_mon(cpu, r); +} + +static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r) +{ + int id = get_domain_id_from_scope(cpu, r->ctrl_scope); + struct rdt_hw_ctrl_domain *hw_dom; + struct rdt_domain_hdr *hdr; + struct rdt_ctrl_domain *d; - BUG_ON(id > NR_CPUS); lockdep_assert_held(&domain_list_lock); - d = rdt_find_domain(r, id, NULL); - if (IS_ERR_OR_NULL(d)) { - pr_warn("Couldn't find cache id for CPU %d\n", cpu); + if (id < 0) { + pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n", + cpu, r->ctrl_scope, r->name); return; } - hw_dom = resctrl_to_arch_dom(d); - cpumask_clear_cpu(cpu, &d->cpu_mask); - if (cpumask_empty(&d->cpu_mask)) { - resctrl_offline_domain(r, d); - list_del_rcu(&d->list); + hdr = resctrl_find_domain(&r->ctrl_domains, id, NULL); + if (!hdr) { + pr_warn("Can't find control domain for id=%d for CPU %d for resource %s\n", + id, cpu, r->name); + return; + } + + if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN)) + return; + + d = container_of(hdr, struct rdt_ctrl_domain, hdr); + hw_dom = resctrl_to_arch_ctrl_dom(d); + + cpumask_clear_cpu(cpu, &d->hdr.cpu_mask); + if (cpumask_empty(&d->hdr.cpu_mask)) { + resctrl_offline_ctrl_domain(r, d); + list_del_rcu(&d->hdr.list); synchronize_rcu(); /* - * rdt_domain "d" is going to be freed below, so clear + * rdt_ctrl_domain "d" is going to be freed below, so clear * its pointer from pseudo_lock_region struct. */ if (d->plr) d->plr->d = NULL; - r->rdt_domain_list[id] = NULL; - domain_free(hw_dom); + ctrl_domain_free(hw_dom); + + return; + } +} + +static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) +{ + int id = get_domain_id_from_scope(cpu, r->mon_scope); + struct rdt_hw_mon_domain *hw_dom; + struct rdt_domain_hdr *hdr; + struct rdt_mon_domain *d; + + lockdep_assert_held(&domain_list_lock); + + if (id < 0) { + pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n", + cpu, r->mon_scope, r->name); + return; + } + + hdr = resctrl_find_domain(&r->mon_domains, id, NULL); + if (!hdr) { + pr_warn("Can't find monitor domain for id=%d for CPU %d for resource %s\n", + id, cpu, r->name); + return; + } + + if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN)) + return; + + d = container_of(hdr, struct rdt_mon_domain, hdr); + hw_dom = resctrl_to_arch_mon_dom(d); + + cpumask_clear_cpu(cpu, &d->hdr.cpu_mask); + if (cpumask_empty(&d->hdr.cpu_mask)) { + resctrl_offline_mon_domain(r, d); + list_del_rcu(&d->hdr.list); + synchronize_rcu(); + mon_domain_free(hw_dom); return; } } +static void domain_remove_cpu(int cpu, struct rdt_resource *r) +{ + if (r->alloc_capable) + domain_remove_cpu_ctrl(cpu, r); + if (r->mon_capable) + domain_remove_cpu_mon(cpu, r); +} + static void clear_closid_rmid(int cpu) { struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state); @@ -643,8 +706,6 @@ enum { RDT_FLAG_MBA, RDT_FLAG_SMBA, RDT_FLAG_BMEC, - RDT_FLAG_ABMC, - RDT_FLAG_SDCIAE, }; #define RDT_OPT(idx, n, f) \ @@ -670,8 +731,6 @@ static struct rdt_options rdt_options[] __ro_after_init = { RDT_OPT(RDT_FLAG_MBA, "mba", X86_FEATURE_MBA), RDT_OPT(RDT_FLAG_SMBA, "smba", X86_FEATURE_SMBA), RDT_OPT(RDT_FLAG_BMEC, "bmec", X86_FEATURE_BMEC), - RDT_OPT(RDT_FLAG_ABMC, "abmc", X86_FEATURE_ABMC), - RDT_OPT(RDT_FLAG_SDCIAE, "sdciae", X86_FEATURE_SDCIAE), }; #define NUM_RDT_OPTIONS ARRAY_SIZE(rdt_options) @@ -745,8 +804,7 @@ static __init bool get_mem_config(void) if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) return __get_mem_config_intel(&hw_res->r_resctrl); - else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) return __rdt_get_mem_config_amd(&hw_res->r_resctrl); return false; @@ -781,8 +839,6 @@ static __init bool get_rdt_alloc_resources(void) rdt_get_cache_alloc_cfg(1, r); if (rdt_cpu_has(X86_FEATURE_CDP_L3)) rdt_get_cdp_l3_config(); - if (rdt_cpu_has(X86_FEATURE_SDCIAE)) - rdt_set_io_alloc_capable(r); ret = true; } if (rdt_cpu_has(X86_FEATURE_CAT_L2)) { @@ -822,18 +878,18 @@ static __init bool get_rdt_mon_resources(void) static __init void __check_quirks_intel(void) { - switch (boot_cpu_data.x86_model) { - case INTEL_FAM6_HASWELL_X: + switch (boot_cpu_data.x86_vfm) { + case INTEL_HASWELL_X: if (!rdt_options[RDT_FLAG_L3_CAT].force_off) cache_alloc_hsw_probe(); break; - case INTEL_FAM6_SKYLAKE_X: + case INTEL_SKYLAKE_X: if (boot_cpu_data.x86_stepping <= 4) set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat"); else set_rdt_options("!l3cat"); fallthrough; - case INTEL_FAM6_BROADWELL_X: + case INTEL_BROADWELL_X: intel_rdt_mbm_apply_quirk(); break; } @@ -899,8 +955,7 @@ static __init void rdt_init_res_defs(void) { if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) rdt_init_res_defs_intel(); - else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) rdt_init_res_defs_amd(); } @@ -931,26 +986,19 @@ void resctrl_cpu_detect(struct cpuinfo_x86 *c) c->x86_cache_occ_scale = ebx; c->x86_cache_mbm_width_offset = eax & 0xff; - if (!c->x86_cache_mbm_width_offset) { - switch (c->x86_vendor) { - case X86_VENDOR_AMD: - c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_AMD; - break; - case X86_VENDOR_HYGON: - c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_HYGON; - break; - default: - /* Leave c->x86_cache_mbm_width_offset as 0 */ - break; - } - } + if (c->x86_vendor == X86_VENDOR_AMD && !c->x86_cache_mbm_width_offset) + c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_AMD; } } static int __init resctrl_arch_late_init(void) { struct rdt_resource *r; - int state, ret; + int state, ret, i; + + /* for_each_rdt_resource() requires all rid to be initialised. */ + for (i = 0; i < RDT_NUM_RESOURCES; i++) + rdt_resources_all[i].r_resctrl.rid = i; /* * Initialize functions(or definitions) that are different diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index b62792ad80acbf081e85da3a07610419796a2563..1189c0df4ad763c176d61b1de7b223f1cea3ac91 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -16,46 +16,27 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include -#include -#include -#include -#include #include "internal.h" -static bool apply_config(struct rdt_hw_domain *hw_dom, - struct resctrl_staged_config *cfg, u32 idx, - cpumask_var_t cpu_mask) -{ - struct rdt_domain *dom = &hw_dom->d_resctrl; - - if (cfg->new_ctrl != hw_dom->ctrl_val[idx]) { - cpumask_set_cpu(cpumask_any(&dom->cpu_mask), cpu_mask); - hw_dom->ctrl_val[idx] = cfg->new_ctrl; - - return true; - } - - return false; -} - -int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, +int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type t, u32 cfg_val) { + struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d); struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); u32 idx = resctrl_get_config_index(closid, t); struct msr_param msr_param; - if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask)) + if (!cpumask_test_cpu(smp_processor_id(), &d->hdr.cpu_mask)) return -EINVAL; hw_dom->ctrl_val[idx] = cfg_val; msr_param.res = r; + msr_param.dom = d; msr_param.low = idx; msr_param.high = idx + 1; - hw_res->msr_update(d, &msr_param, r); + hw_res->msr_update(&msr_param); return 0; } @@ -63,99 +44,50 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) { struct resctrl_staged_config *cfg; - struct rdt_hw_domain *hw_dom; + struct rdt_hw_ctrl_domain *hw_dom; struct msr_param msr_param; + struct rdt_ctrl_domain *d; enum resctrl_conf_type t; - cpumask_var_t cpu_mask; - struct rdt_domain *d; u32 idx; /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL)) - return -ENOMEM; - - msr_param.res = NULL; - list_for_each_entry(d, &r->domains, list) { - hw_dom = resctrl_to_arch_dom(d); + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + hw_dom = resctrl_to_arch_ctrl_dom(d); + msr_param.res = NULL; for (t = 0; t < CDP_NUM_TYPES; t++) { cfg = &hw_dom->d_resctrl.staged_config[t]; if (!cfg->have_new_ctrl) continue; idx = resctrl_get_config_index(closid, t); - if (!apply_config(hw_dom, cfg, idx, cpu_mask)) + if (cfg->new_ctrl == hw_dom->ctrl_val[idx]) continue; + hw_dom->ctrl_val[idx] = cfg->new_ctrl; if (!msr_param.res) { msr_param.low = idx; msr_param.high = msr_param.low + 1; msr_param.res = r; + msr_param.dom = d; } else { msr_param.low = min(msr_param.low, idx); msr_param.high = max(msr_param.high, idx + 1); } } + if (msr_param.res) + smp_call_function_any(&d->hdr.cpu_mask, rdt_ctrl_update, &msr_param, 1); } - if (cpumask_empty(cpu_mask)) - goto done; - - /* Update resource control msr on all the CPUs. */ - on_each_cpu_mask(cpu_mask, rdt_ctrl_update, &msr_param, 1); - -done: - free_cpumask_var(cpu_mask); - return 0; } -u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d, +u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type type) { - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d); u32 idx = resctrl_get_config_index(closid, type); return hw_dom->ctrl_val[idx]; } - -bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r) -{ - return resctrl_to_arch_res(r)->sdciae_enabled; -} - -static void resctrl_sdciae_set_one_amd(void *arg) -{ - bool *enable = arg; - - if (*enable) - msr_set_bit(MSR_IA32_L3_QOS_EXT_CFG, SDCIAE_ENABLE_BIT); - else - msr_clear_bit(MSR_IA32_L3_QOS_EXT_CFG, SDCIAE_ENABLE_BIT); -} - -static void _resctrl_sdciae_enable(struct rdt_resource *r, bool enable) -{ - struct rdt_domain *d; - - /* Walking r->ctrl_domains, ensure it can't race with cpuhp */ - lockdep_assert_cpus_held(); - - /* Update MSR_IA32_L3_QOS_EXT_CFG MSR on all the CPUs in all domains */ - list_for_each_entry(d, &r->domains, list) - on_each_cpu_mask(&d->cpu_mask, resctrl_sdciae_set_one_amd, &enable, 1); -} - -int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable) -{ - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - - if (hw_res->r_resctrl.cache.io_alloc_capable && - hw_res->sdciae_enabled != enable) { - _resctrl_sdciae_enable(r, enable); - hw_res->sdciae_enabled = enable; - } - - return 0; -} diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 41c912aa859cb8d80bc932a937c98e8d624cab1b..5e3c41b3643737e25110d1e91a3ab17f13cc2f06 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -3,34 +3,19 @@ #define _ASM_X86_RESCTRL_INTERNAL_H #include -#include -#include -#include -#include -#include - -#include -#include - -/* Memory bandwidth HWDRC */ -#define HWDRC_MSR_OS_MAILBOX_INTERFACE 0xb0 -#define HWDRC_MSR_OS_MAILBOX_DATA 0xb1 -#define HWDRC_MSR_OS_MAILBOX_BUSY_BIT BIT_ULL(31) -#define HWDRC_COMMAND_MEM_CLOS_EN 0xd0 -#define HWDRC_SUB_COMMAND_MEM_CLOS_EN 0x54 -#define HWDRC_MEMCLOS_AVAILABLE BIT_ULL(0) -#define HWDRC_OS_MAILBOX_RETRY_COUNT 30 #define L3_QOS_CDP_ENABLE 0x01ULL #define L2_QOS_CDP_ENABLE 0x01ULL -#define MBM_CNTR_WIDTH_OFFSET_AMD 20 +#define MBM_CNTR_WIDTH_BASE 24 + +#define MBA_IS_LINEAR 0x4 -/* Hygon MBM counter width as an offset from MBM_CNTR_WIDTH_BASE */ -#define MBM_CNTR_WIDTH_OFFSET_HYGON 8 +#define MBM_CNTR_WIDTH_OFFSET_AMD 20 #define RMID_VAL_ERROR BIT_ULL(63) + #define RMID_VAL_UNAVAIL BIT_ULL(62) /* @@ -40,9 +25,6 @@ */ #define MBM_CNTR_WIDTH_OFFSET_MAX (62 - MBM_CNTR_WIDTH_BASE) -/* Setting bit 0 in L3_QOS_EXT_CFG enables the ABMC feature. */ -#define ABMC_ENABLE_BIT 0 - /** * struct arch_mbm_state - values used to compute resctrl_arch_rmid_read()s * return value. @@ -55,43 +37,54 @@ struct arch_mbm_state { u64 prev_msr; }; -/* Setting bit 1 in MSR_IA32_L3_QOS_EXT_CFG enables the SDCIAE feature. */ -#define SDCIAE_ENABLE_BIT 1 - /** - * struct rdt_hw_domain - Arch private attributes of a set of CPUs that share - * a resource + * struct rdt_hw_ctrl_domain - Arch private attributes of a set of CPUs that share + * a resource for a control function * @d_resctrl: Properties exposed to the resctrl file system * @ctrl_val: array of cache or mem ctrl values (indexed by CLOSID) + * + * Members of this structure are accessed via helpers that provide abstraction. + */ +struct rdt_hw_ctrl_domain { + struct rdt_ctrl_domain d_resctrl; + u32 *ctrl_val; +}; + +/** + * struct rdt_hw_mon_domain - Arch private attributes of a set of CPUs that share + * a resource for a monitor function + * @d_resctrl: Properties exposed to the resctrl file system * @arch_mbm_total: arch private state for MBM total bandwidth * @arch_mbm_local: arch private state for MBM local bandwidth - * @mbm_total_cfg: MBM total bandwidth configuration - * @mbm_local_cfg: MBM local bandwidth configuration * * Members of this structure are accessed via helpers that provide abstraction. */ -struct rdt_hw_domain { - struct rdt_domain d_resctrl; - u32 *ctrl_val; +struct rdt_hw_mon_domain { + struct rdt_mon_domain d_resctrl; struct arch_mbm_state *arch_mbm_total; struct arch_mbm_state *arch_mbm_local; - u32 mbm_total_cfg; - u32 mbm_local_cfg; }; -static inline struct rdt_hw_domain *resctrl_to_arch_dom(struct rdt_domain *r) +static inline struct rdt_hw_ctrl_domain *resctrl_to_arch_ctrl_dom(struct rdt_ctrl_domain *r) { - return container_of(r, struct rdt_hw_domain, d_resctrl); + return container_of(r, struct rdt_hw_ctrl_domain, d_resctrl); +} + +static inline struct rdt_hw_mon_domain *resctrl_to_arch_mon_dom(struct rdt_mon_domain *r) +{ + return container_of(r, struct rdt_hw_mon_domain, d_resctrl); } /** * struct msr_param - set a range of MSRs from a domain * @res: The resource to use + * @dom: The domain to update * @low: Beginning index from base MSR * @high: End index */ struct msr_param { struct rdt_resource *res; + struct rdt_ctrl_domain *dom; u32 low; u32 high; }; @@ -109,8 +102,6 @@ struct msr_param { * @mon_scale: cqm counter * mon_scale = occupancy in bytes * @mbm_width: Monitor width, to detect and correct for overflow. * @cdp_enabled: CDP state of this resource - * @mbm_cntr_assign_enabled: ABMC feature is enabled - * @sdciae_enabled: SDCIAE feature (backing "io_alloc") is enabled. * * Members of this structure are either private to the architecture * e.g. mbm_width, or accessed via helpers that provide abstraction. e.g. @@ -120,13 +111,10 @@ struct rdt_hw_resource { struct rdt_resource r_resctrl; u32 num_closid; unsigned int msr_base; - void (*msr_update) (struct rdt_domain *d, struct msr_param *m, - struct rdt_resource *r); + void (*msr_update)(struct msr_param *m); unsigned int mon_scale; unsigned int mbm_width; bool cdp_enabled; - bool mbm_cntr_assign_enabled; - bool sdciae_enabled; }; static inline struct rdt_hw_resource *resctrl_to_arch_res(struct rdt_resource *r) @@ -136,34 +124,7 @@ static inline struct rdt_hw_resource *resctrl_to_arch_res(struct rdt_resource *r extern struct rdt_hw_resource rdt_resources_all[]; -static inline struct rdt_resource *resctrl_inc(struct rdt_resource *res) -{ - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(res); - - hw_res++; - return &hw_res->r_resctrl; -} - -/* - * To return the common struct rdt_resource, which is contained in struct - * rdt_hw_resource, walk the resctrl member of struct rdt_hw_resource. - */ -#define for_each_rdt_resource(r) \ - for (r = &rdt_resources_all[0].r_resctrl; \ - r <= &rdt_resources_all[RDT_NUM_RESOURCES - 1].r_resctrl; \ - r = resctrl_inc(r)) - -#define for_each_capable_rdt_resource(r) \ - for_each_rdt_resource(r) \ - if (r->alloc_capable || r->mon_capable) - -#define for_each_alloc_capable_rdt_resource(r) \ - for_each_rdt_resource(r) \ - if (r->alloc_capable) - -#define for_each_mon_capable_rdt_resource(r) \ - for_each_rdt_resource(r) \ - if (r->mon_capable) +void arch_mon_domain_online(struct rdt_resource *r, struct rdt_mon_domain *d); /* CPUID.(EAX=10H, ECX=ResID=1).EAX */ union cpuid_0x10_1_eax { @@ -198,36 +159,14 @@ union cpuid_0x10_x_edx { unsigned int full; }; -/* - * ABMC counters can be configured by writing to L3_QOS_ABMC_CFG. - * @bw_type : Bandwidth configuration(supported by BMEC) - * tracked by the @cntr_id. - * @bw_src : Bandwidth source (RMID or CLOSID). - * @reserved1 : Reserved. - * @is_clos : @bw_src field is a CLOSID (not an RMID). - * @cntr_id : Counter identifier. - * @reserved : Reserved. - * @cntr_en : Tracking enable bit. - * @cfg_en : Configuration enable bit. - */ -union l3_qos_abmc_cfg { - struct { - unsigned long bw_type :32, - bw_src :12, - reserved1: 3, - is_clos : 1, - cntr_id : 5, - reserved : 9, - cntr_en : 1, - cfg_en : 1; - } split; - unsigned long full; -}; - void rdt_ctrl_update(void *arg); + int rdt_get_mon_l3_config(struct rdt_resource *r); + bool rdt_cpu_has(int flag); + void __init intel_rdt_mbm_apply_quirk(void); + void rdt_domain_reconfigure_cdp(struct rdt_resource *r); -void resctrl_mbm_evt_config_init(struct rdt_hw_domain *hw_dom); + #endif /* _ASM_X86_RESCTRL_INTERNAL_H */ diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 0ed28ef8c25a791733325b56cb6fd90100b33d7e..60de452c951134317a99c6a3b3827e598545645f 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -15,13 +15,12 @@ * Software Developer Manual June 2016, volume 3, section 17.17. */ +#define pr_fmt(fmt) "resctrl: " fmt + #include -#include -#include -#include +#include #include -#include #include "internal.h" @@ -38,8 +37,10 @@ unsigned int rdt_mon_features; #define CF(cf) ((unsigned long)(1048576 * (cf) + 0.5)) +static int snc_nodes_per_l3_cache = 1; + /* - * The correction factor table is documented in Documentation/arch/x86/resctrl.rst. + * The correction factor table is documented in Documentation/filesystems/resctrl.rst. * If rmid > rmid threshold, MBM total and local values should be multiplied * by the correction factor. * @@ -88,6 +89,7 @@ static const struct mbm_correction_factor_table { }; static u32 mbm_cf_rmidthreshold __read_mostly = UINT_MAX; + static u64 mbm_cf __read_mostly; static inline u64 get_corrected_mbm_count(u32 rmid, unsigned long val) @@ -99,7 +101,43 @@ static inline u64 get_corrected_mbm_count(u32 rmid, unsigned long val) return val; } -static int __rmid_read(u32 rmid, enum resctrl_event_id eventid, u64 *val) +/* + * When Sub-NUMA Cluster (SNC) mode is not enabled (as indicated by + * "snc_nodes_per_l3_cache == 1") no translation of the RMID value is + * needed. The physical RMID is the same as the logical RMID. + * + * On a platform with SNC mode enabled, Linux enables RMID sharing mode + * via MSR 0xCA0 (see the "RMID Sharing Mode" section in the "Intel + * Resource Director Technology Architecture Specification" for a full + * description of RMID sharing mode). + * + * In RMID sharing mode there are fewer "logical RMID" values available + * to accumulate data ("physical RMIDs" are divided evenly between SNC + * nodes that share an L3 cache). Linux creates an rdt_mon_domain for + * each SNC node. + * + * The value loaded into IA32_PQR_ASSOC is the "logical RMID". + * + * Data is collected independently on each SNC node and can be retrieved + * using the "physical RMID" value computed by this function and loaded + * into IA32_QM_EVTSEL. @cpu can be any CPU in the SNC node. + * + * The scope of the IA32_QM_EVTSEL and IA32_QM_CTR MSRs is at the L3 + * cache. So a "physical RMID" may be read from any CPU that shares + * the L3 cache with the desired SNC node, not just from a CPU in + * the specific SNC node. + */ +static int logical_rmid_to_physical_rmid(int cpu, int lrmid) +{ + struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; + + if (snc_nodes_per_l3_cache == 1) + return lrmid; + + return lrmid + (cpu_to_node(cpu) % snc_nodes_per_l3_cache) * r->num_rmid; +} + +static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) { u64 msr_val; @@ -111,7 +149,7 @@ static int __rmid_read(u32 rmid, enum resctrl_event_id eventid, u64 *val) * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62) * are error bits. */ - wrmsr(MSR_IA32_QM_EVTSEL, eventid, rmid); + wrmsr(MSR_IA32_QM_EVTSEL, eventid, prmid); rdmsrl(MSR_IA32_QM_CTR, msr_val); if (msr_val & RMID_VAL_ERROR) @@ -123,7 +161,7 @@ static int __rmid_read(u32 rmid, enum resctrl_event_id eventid, u64 *val) return 0; } -static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_domain *hw_dom, +static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_mon_domain *hw_dom, u32 rmid, enum resctrl_event_id eventid) { @@ -135,28 +173,28 @@ static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_domain *hw_dom, case QOS_L3_MBM_LOCAL_EVENT_ID: return &hw_dom->arch_mbm_local[rmid]; default: - break; + /* Never expect to get here */ + WARN_ON_ONCE(1); + return NULL; } - - /* Never expect to get here */ - WARN_ON_ONCE(1); - - return NULL; } -void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d, u32 unused, u32 rmid, enum resctrl_event_id eventid) { - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); + int cpu = cpumask_any(&d->hdr.cpu_mask); struct arch_mbm_state *am; + u32 prmid; am = get_arch_mbm_state(hw_dom, rmid, eventid); if (am) { memset(am, 0, sizeof(*am)); + prmid = logical_rmid_to_physical_rmid(cpu, rmid); /* Record any initial, non-zero count value. */ - __rmid_read(rmid, eventid, &am->prev_msr); + __rmid_read_phys(prmid, eventid, &am->prev_msr); } } @@ -164,17 +202,17 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, * Assumes that hardware counters are also reset and thus that there is * no need to record initial non-zero counts. */ -void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_domain *d) +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain *d) { - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); if (resctrl_arch_is_mbm_total_enabled()) memset(hw_dom->arch_mbm_total, 0, - sizeof(*hw_dom->arch_mbm_total) * r->mon.num_rmid); + sizeof(*hw_dom->arch_mbm_total) * r->num_rmid); if (resctrl_arch_is_mbm_local_enabled()) memset(hw_dom->arch_mbm_local, 0, - sizeof(*hw_dom->arch_mbm_local) * r->mon.num_rmid); + sizeof(*hw_dom->arch_mbm_local) * r->num_rmid); } static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width) @@ -185,22 +223,22 @@ static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width) return chunks >> shift; } -int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_mon_domain *d, u32 unused, u32 rmid, enum resctrl_event_id eventid, u64 *val, void *ignored) { + struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + int cpu = cpumask_any(&d->hdr.cpu_mask); struct arch_mbm_state *am; u64 msr_val, chunks; + u32 prmid; int ret; resctrl_arch_rmid_read_context_check(); - if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask)) - return -EINVAL; - - ret = __rmid_read(rmid, eventid, &msr_val); + prmid = logical_rmid_to_physical_rmid(cpu, rmid); + ret = __rmid_read_phys(prmid, eventid, &msr_val); if (ret) return ret; @@ -219,16 +257,101 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, return 0; } +/* + * The power-on reset value of MSR_RMID_SNC_CONFIG is 0x1 + * which indicates that RMIDs are configured in legacy mode. + * This mode is incompatible with Linux resctrl semantics + * as RMIDs are partitioned between SNC nodes, which requires + * a user to know which RMID is allocated to a task. + * Clearing bit 0 reconfigures the RMID counters for use + * in RMID sharing mode. This mode is better for Linux. + * The RMID space is divided between all SNC nodes with the + * RMIDs renumbered to start from zero in each node when + * counting operations from tasks. Code to read the counters + * must adjust RMID counter numbers based on SNC node. See + * logical_rmid_to_physical_rmid() for code that does this. + */ +void arch_mon_domain_online(struct rdt_resource *r, struct rdt_mon_domain *d) +{ + if (snc_nodes_per_l3_cache > 1) + msr_clear_bit(MSR_RMID_SNC_CONFIG, 0); +} + +/* CPU models that support MSR_RMID_SNC_CONFIG */ +static const struct x86_cpu_id snc_cpu_ids[] __initconst = { + X86_MATCH_VFM(INTEL_ICELAKE_X, 0), + X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, 0), + X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, 0), + X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, 0), + X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, 0), + X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, 0), + {} +}; + +/* + * There isn't a simple hardware bit that indicates whether a CPU is running + * in Sub-NUMA Cluster (SNC) mode. Infer the state by comparing the + * number of CPUs sharing the L3 cache with CPU0 to the number of CPUs in + * the same NUMA node as CPU0. + * It is not possible to accurately determine SNC state if the system is + * booted with a maxcpus=N parameter. That distorts the ratio of SNC nodes + * to L3 caches. It will be OK if system is booted with hyperthreading + * disabled (since this doesn't affect the ratio). + */ +static __init int snc_get_config(void) +{ + struct cacheinfo *ci = get_cpu_cacheinfo_level(0, RESCTRL_L3_CACHE); + const cpumask_t *node0_cpumask; + int cpus_per_node, cpus_per_l3; + int ret; + + if (!x86_match_cpu(snc_cpu_ids) || !ci) + return 1; + + cpus_read_lock(); + if (num_online_cpus() != num_present_cpus()) + pr_warn("Some CPUs offline, SNC detection may be incorrect\n"); + cpus_read_unlock(); + + node0_cpumask = cpumask_of_node(cpu_to_node(0)); + + cpus_per_node = cpumask_weight(node0_cpumask); + cpus_per_l3 = cpumask_weight(&ci->shared_cpu_map); + + if (!cpus_per_node || !cpus_per_l3) + return 1; + + ret = cpus_per_l3 / cpus_per_node; + + /* sanity check: Only valid results are 1, 2, 3, 4, 6 */ + switch (ret) { + case 1: + break; + case 2 ... 4: + case 6: + pr_info("Sub-NUMA Cluster mode detected with %d nodes per L3 cache\n", ret); + rdt_resources_all[RDT_RESOURCE_L3].r_resctrl.mon_scope = RESCTRL_L3_NODE; + break; + default: + pr_warn("Ignore improbable SNC node count %d\n", ret); + ret = 1; + break; + } + + return ret; +} + int __init rdt_get_mon_l3_config(struct rdt_resource *r) { unsigned int mbm_offset = boot_cpu_data.x86_cache_mbm_width_offset; struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); unsigned int threshold; - u32 eax, ebx, ecx, edx; + + snc_nodes_per_l3_cache = snc_get_config(); resctrl_rmid_realloc_limit = boot_cpu_data.x86_cache_size * 1024; - hw_res->mon_scale = boot_cpu_data.x86_cache_occ_scale; - r->mon.num_rmid = boot_cpu_data.x86_cache_max_rmid + 1; + hw_res->mon_scale = boot_cpu_data.x86_cache_occ_scale / snc_nodes_per_l3_cache; + r->num_rmid = (boot_cpu_data.x86_cache_max_rmid + 1) / snc_nodes_per_l3_cache; hw_res->mbm_width = MBM_CNTR_WIDTH_BASE; if (mbm_offset > 0 && mbm_offset <= MBM_CNTR_WIDTH_OFFSET_MAX) @@ -243,22 +366,7 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r) * * For a 35MB LLC and 56 RMIDs, this is ~1.8% of the LLC. */ - threshold = resctrl_rmid_realloc_limit / r->mon.num_rmid; - - if (rdt_cpu_has(X86_FEATURE_ABMC)) { - r->mon.mbm_cntr_assignable = true; - /* - * Query CPUID_Fn80000020_EBX_x05 for number of - * ABMC counters. - */ - cpuid_count(0x80000020, 5, &eax, &ebx, &ecx, &edx); - r->mon.num_mbm_cntrs = (ebx & 0xFFFF) + 1; - if (WARN_ON(r->mon.num_mbm_cntrs > 64)) - r->mon.num_mbm_cntrs = 64; - - resctrl_file_fflags_init("num_mbm_cntrs", RFTYPE_MON_INFO); - resctrl_file_fflags_init("mbm_control", RFTYPE_MON_INFO); - } + threshold = resctrl_rmid_realloc_limit / r->num_rmid; /* * Because num_rmid may not be a power of two, round the value @@ -280,32 +388,6 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r) return 0; } -void resctrl_mbm_evt_config_init(struct rdt_hw_domain *hw_dom) -{ - unsigned int index; - u64 msrval; - - /* - * Read the configuration registers QOS_EVT_CFG_n, where is - * the BMEC event number (EvtID). - */ - if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_TOTAL_EVENT_ID)) { - index = mon_event_config_index_get(QOS_L3_MBM_TOTAL_EVENT_ID); - rdmsrl(MSR_IA32_EVT_CFG_BASE + index, msrval); - hw_dom->mbm_total_cfg = msrval & MAX_EVT_CONFIG_BITS; - } else { - hw_dom->mbm_total_cfg = INVALID_CONFIG_VALUE; - } - - if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_LOCAL_EVENT_ID)) { - index = mon_event_config_index_get(QOS_L3_MBM_LOCAL_EVENT_ID); - rdmsrl(MSR_IA32_EVT_CFG_BASE + index, msrval); - hw_dom->mbm_local_cfg = msrval & MAX_EVT_CONFIG_BITS; - } else { - hw_dom->mbm_total_cfg = INVALID_CONFIG_VALUE; - } -} - void __init intel_rdt_mbm_apply_quirk(void) { int cf_index; diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index ba1596afee107f65f784ee004a86d0faabf68eb4..2e13473c5aa4e5ee0c78e12f6d1499a6d41584af 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -11,27 +11,21 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include +#include #include -#include -#include -#include -#include #include #include -#include -#include +#include -#include -#include -#include +#include #include #include "../../events/perf_event.h" /* For X86_CONFIG() */ #include "internal.h" #define CREATE_TRACE_POINTS -#include "pseudo_lock_event.h" + +#include "pseudo_lock_trace.h" /* * The bits needed to disable hardware prefetching varies based on the @@ -69,8 +63,8 @@ u64 resctrl_arch_get_prefetch_disable_bits(void) boot_cpu_data.x86 != 6) return 0; - switch (boot_cpu_data.x86_model) { - case INTEL_FAM6_BROADWELL_X: + switch (boot_cpu_data.x86_vfm) { + case INTEL_BROADWELL_X: /* * SDM defines bits of MSR_MISC_FEATURE_CONTROL register * as: @@ -82,8 +76,8 @@ u64 resctrl_arch_get_prefetch_disable_bits(void) */ prefetch_disable_bits = 0xF; break; - case INTEL_FAM6_ATOM_GOLDMONT: - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: + case INTEL_ATOM_GOLDMONT: + case INTEL_ATOM_GOLDMONT_PLUS: /* * SDM defines bits of MSR_MISC_FEATURE_CONTROL register * as: @@ -440,9 +434,9 @@ int resctrl_arch_measure_l2_residency(void *_plr) * L2_HIT 02H * L2_MISS 10H */ - switch (boot_cpu_data.x86_model) { - case INTEL_FAM6_ATOM_GOLDMONT: - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: + switch (boot_cpu_data.x86_vfm) { + case INTEL_ATOM_GOLDMONT: + case INTEL_ATOM_GOLDMONT_PLUS: perf_miss_attr.config = X86_CONFIG(.event = 0xd1, .umask = 0x10); perf_hit_attr.config = X86_CONFIG(.event = 0xd1, @@ -479,8 +473,8 @@ int resctrl_arch_measure_l3_residency(void *_plr) * MISS 41H */ - switch (boot_cpu_data.x86_model) { - case INTEL_FAM6_BROADWELL_X: + switch (boot_cpu_data.x86_vfm) { + case INTEL_BROADWELL_X: /* On BDW the hit event counts references, not hits */ perf_hit_attr.config = X86_CONFIG(.event = 0x2e, .umask = 0x4f); @@ -498,7 +492,7 @@ int resctrl_arch_measure_l3_residency(void *_plr) */ counts.miss_after -= counts.miss_before; - if (boot_cpu_data.x86_model == INTEL_FAM6_BROADWELL_X) { + if (boot_cpu_data.x86_vfm == INTEL_BROADWELL_X) { /* * On BDW references and misses are counted, need to adjust. * Sometimes the "hits" counter is a bit more than the diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock_event.h b/arch/x86/kernel/cpu/resctrl/pseudo_lock_trace.h similarity index 83% rename from arch/x86/kernel/cpu/resctrl/pseudo_lock_event.h rename to arch/x86/kernel/cpu/resctrl/pseudo_lock_trace.h index 428ebbd4270b93e482b4af78d1c2c8bff5b27cca..7c8aef08010f1431a2cfcfb96a32873cd03fc024 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock_event.h +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock_trace.h @@ -2,8 +2,8 @@ #undef TRACE_SYSTEM #define TRACE_SYSTEM resctrl -#if !defined(_TRACE_PSEUDO_LOCK_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_PSEUDO_LOCK_H +#if !defined(_X86_RESCTRL_PSEUDO_LOCK_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _X86_RESCTRL_PSEUDO_LOCK_TRACE_H #include @@ -35,9 +35,11 @@ TRACE_EVENT(pseudo_lock_l3, TP_printk("hits=%llu miss=%llu", __entry->l3_hits, __entry->l3_miss)); -#endif /* _TRACE_PSEUDO_LOCK_H */ +#endif /* _X86_RESCTRL_PSEUDO_LOCK_TRACE_H */ #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH . -#define TRACE_INCLUDE_FILE pseudo_lock_event + +#define TRACE_INCLUDE_FILE pseudo_lock_trace + #include diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 50a49a1ea604d842cedb551a310f2e3c893e0c5c..c7a7f0ae373adf1a7a348f1f8b37dc05c570a10d 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -13,13 +13,28 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include + +#include -#include #include "internal.h" DEFINE_STATIC_KEY_FALSE(rdt_enable_key); + DEFINE_STATIC_KEY_FALSE(rdt_mon_enable_key); + DEFINE_STATIC_KEY_FALSE(rdt_alloc_enable_key); /* @@ -28,9 +43,9 @@ DEFINE_STATIC_KEY_FALSE(rdt_alloc_enable_key); * from update_closid_rmid() is protected against __switch_to() because * preemption is disabled. */ -void resctrl_arch_sync_cpu_defaults(void *info) +void resctrl_arch_sync_cpu_closid_rmid(void *info) { - struct resctrl_cpu_sync *r = info; + struct resctrl_cpu_defaults *r = info; if (r) { this_cpu_write(pqr_state.default_closid, r->closid); @@ -45,37 +60,58 @@ void resctrl_arch_sync_cpu_defaults(void *info) resctrl_arch_sched_in(current); } -void resctrl_arch_mon_event_config_read(void *info) +#define INVALID_CONFIG_INDEX UINT_MAX + +/** + * mon_event_config_index_get - get the hardware index for the + * configurable event + * @evtid: event id. + * + * Return: 0 for evtid == QOS_L3_MBM_TOTAL_EVENT_ID + * 1 for evtid == QOS_L3_MBM_LOCAL_EVENT_ID + * INVALID_CONFIG_INDEX for invalid evtid + */ +static inline unsigned int mon_event_config_index_get(u32 evtid) +{ + switch (evtid) { + case QOS_L3_MBM_TOTAL_EVENT_ID: + return 0; + case QOS_L3_MBM_LOCAL_EVENT_ID: + return 1; + default: + /* Should never reach here */ + return INVALID_CONFIG_INDEX; + } +} + +void resctrl_arch_mon_event_config_read(void *_config_info) { - struct resctrl_mon_config_info *mon_info = info; + struct resctrl_mon_config_info *config_info = _config_info; unsigned int index; u64 msrval; - index = mon_event_config_index_get(mon_info->evtid); + index = mon_event_config_index_get(config_info->evtid); if (index == INVALID_CONFIG_INDEX) { - pr_warn_once("Invalid event id %d\n", mon_info->evtid); + pr_warn_once("Invalid event id %d\n", config_info->evtid); return; } rdmsrl(MSR_IA32_EVT_CFG_BASE + index, msrval); /* Report only the valid event configuration bits */ - mon_info->mon_config = msrval & MAX_EVT_CONFIG_BITS; + config_info->mon_config = msrval & MAX_EVT_CONFIG_BITS; } -void resctrl_arch_mon_event_config_write(void *info) +void resctrl_arch_mon_event_config_write(void *_config_info) { - struct resctrl_mon_config_info *mon_info = info; + struct resctrl_mon_config_info *config_info = _config_info; unsigned int index; - index = mon_event_config_index_get(mon_info->evtid); + index = mon_event_config_index_get(config_info->evtid); if (index == INVALID_CONFIG_INDEX) { - pr_warn_once("Invalid event id %d\n", mon_info->evtid); - mon_info->err = -EINVAL; + pr_warn_once("Invalid event id %d\n", config_info->evtid); return; } - wrmsr(MSR_IA32_EVT_CFG_BASE + index, mon_info->mon_config, 0); - - mon_info->err = 0; + wrmsr(MSR_IA32_EVT_CFG_BASE + index, config_info->mon_config, 0); } static void l3_qos_cfg_update(void *arg) @@ -95,9 +131,9 @@ static void l2_qos_cfg_update(void *arg) static int set_cache_qos_cfg(int level, bool enable) { void (*update)(void *arg); + struct rdt_ctrl_domain *d; struct rdt_resource *r_l; cpumask_var_t cpu_mask; - struct rdt_domain *d; int cpu; /* Walking r->domains, ensure it can't race with cpuhp */ @@ -114,14 +150,14 @@ static int set_cache_qos_cfg(int level, bool enable) return -ENOMEM; r_l = &rdt_resources_all[level].r_resctrl; - list_for_each_entry(d, &r_l->domains, list) { + list_for_each_entry(d, &r_l->ctrl_domains, hdr.list) { if (r_l->cache.arch_has_per_cpu_cfg) /* Pick all the CPUs in the domain instance */ - for_each_cpu(cpu, &d->cpu_mask) + for_each_cpu(cpu, &d->hdr.cpu_mask) cpumask_set_cpu(cpu, cpu_mask); else /* Pick one CPU from each domain instance to update MSR */ - cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask); + cpumask_set_cpu(cpumask_any(&d->hdr.cpu_mask), cpu_mask); } /* Update QOS_CFG MSR on all the CPUs in cpu_mask */ @@ -187,356 +223,39 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable) return 0; } -/* - * Workaround to detect if memory bandwidth HWDRC feature is capable. - * - * CPUID for memory bandwidth HWDRC feature is not exposed by H/W. - * Check presence of HWDRC OS mailbox MSRs. Read out the discovery bit of - * HWDRC OS mailbox data which indicates if the feature is capable. - */ -bool resctrl_arch_is_hwdrc_mb_capable(void) -{ - u32 retries; - u64 data; - int status; - - /* Only enable memory bandwidth HWDRC after ICELAKE server */ - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || - boot_cpu_data.x86_model < INTEL_FAM6_ICELAKE_X) { - pr_debug("HWDRC: Not support until ICELAKE server\n"); - goto out; - } - - /* Check presence of mailbox MSRs */ - if (rdmsrl_safe(HWDRC_MSR_OS_MAILBOX_INTERFACE, &data)) { - pr_debug("HWDRC: Can't access OS mailbox interface MSR\n"); - goto out; - } - - if (rdmsrl_safe(HWDRC_MSR_OS_MAILBOX_DATA, &data)) { - pr_debug("HWDRC: Can't access OS mailbox data MSR\n"); - goto out; - } - - /* Poll for run_busy bit == 0 */ - status = -EBUSY; - retries = HWDRC_OS_MAILBOX_RETRY_COUNT; - do { - rdmsrl(HWDRC_MSR_OS_MAILBOX_INTERFACE, data); - if (!(data & HWDRC_MSR_OS_MAILBOX_BUSY_BIT)) { - status = 0; - break; - } - } while (--retries); - - if (status) - goto out; - - /* Write command register: 0x800054d0 */ - data = HWDRC_MSR_OS_MAILBOX_BUSY_BIT | - HWDRC_SUB_COMMAND_MEM_CLOS_EN << 8 | - HWDRC_COMMAND_MEM_CLOS_EN; - pr_debug("HWDRC: Write command register: 0x%llx\n", data); - if (wrmsrl_safe(HWDRC_MSR_OS_MAILBOX_INTERFACE, data)) { - pr_debug("HWDRC: Write command register 0x%llx failed!\n", data); - goto out; - } - - /* Poll for run_busy bit == 0 */ - retries = HWDRC_OS_MAILBOX_RETRY_COUNT; - do { - rdmsrl(HWDRC_MSR_OS_MAILBOX_INTERFACE, data); - if (!(data & HWDRC_MSR_OS_MAILBOX_BUSY_BIT)) { - rdmsrl(HWDRC_MSR_OS_MAILBOX_DATA, data); - pr_debug("HWDRC: Read MEM_CLOS_EN data: 0x%llx\n", data); - - /* Feature capability bit is set */ - if (data & HWDRC_MEMCLOS_AVAILABLE) { - pr_debug("HWDRC: Memory bandwidth HWDRC is capable\n"); - return true; - } - - /* Feature capability bit is not set */ - break; - } - } while (--retries); - -out: - pr_debug("HWDRC: Memory bandwidth HWDRC is not capable\n"); - return false; -} - -static void mba_enable(enum resctrl_res_level l) -{ - struct rdt_hw_resource *r_hw = &rdt_resources_all[l]; - struct rdt_resource *r = &r_hw->r_resctrl; - - r->alloc_capable = true; -} - -static void mba_disable(enum resctrl_res_level l) -{ - struct rdt_hw_resource *r_hw = &rdt_resources_all[l]; - struct rdt_resource *r = &r_hw->r_resctrl; - - r->alloc_capable = false; -} - -/* - * Currently memory bandwidth HWDRC feature is enabled or disabled by the user - * outside of the scope of the resctrl filesystem. When memory bandwidth HWDRC - * is enabled, it takes over MBA hooks in resctrl for memory bandwidth - * throttling. - * - * Set memory bandwidth HWDRC enabled in resctrl so that the user who enables - * memory bandwidth HWDRC can make sure that resctrl doesn't provide any hooks - * to control MBA. - * - * Set memory bandwidth HWDRC disabled in resctrl, MBA is enabled by default. - */ -int resctrl_arch_set_hwdrc_enabled(enum resctrl_res_level l, bool hwdrc_mb) -{ - struct rdt_resource *r = &rdt_resources_all[l].r_resctrl; - - if (!resctrl_arch_is_hwdrc_mb_capable() || hwdrc_mb == r->membw.hwdrc_mb) - return -EINVAL; - - /* MBA and memory bandwidth HWDRC features are mutually exclusive */ - if (hwdrc_mb) - mba_disable(l); - else - mba_enable(l); - - r->membw.hwdrc_mb = hwdrc_mb; - - return 0; -} - -/* - * Update L3_QOS_EXT_CFG MSR on all the CPUs associated with the resource. - */ -static void resctrl_abmc_set_one_amd(void *arg) -{ - bool *enable = arg; - - if (*enable) - msr_set_bit(MSR_IA32_L3_QOS_EXT_CFG, ABMC_ENABLE_BIT); - else - msr_clear_bit(MSR_IA32_L3_QOS_EXT_CFG, ABMC_ENABLE_BIT); -} - -static void _resctrl_abmc_enable(struct rdt_resource *r, bool enable) -{ - struct rdt_domain *d; - - /* - * Hardware counters will reset after switching the monitor mode. - * Reset the architectural state so that reading of hardware - * counter is not considered as an overflow in the next update. - */ - list_for_each_entry(d, &r->domains, list) { - on_each_cpu_mask(&d->cpu_mask, - resctrl_abmc_set_one_amd, &enable, 1); - resctrl_arch_reset_rmid_all(r, d); - } -} - -bool resctrl_arch_get_abmc_enabled(void) +bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level l) { - return rdt_resources_all[RDT_RESOURCE_L3].mbm_cntr_assign_enabled; + return rdt_resources_all[l].cdp_enabled; } -int resctrl_arch_mbm_cntr_assign_enable(void) +void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) { - struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - - lockdep_assert_held(&rdtgroup_mutex); - - if (r->mon.mbm_cntr_assignable && !hw_res->mbm_cntr_assign_enabled) { - _resctrl_abmc_enable(r, true); - hw_res->mbm_cntr_assign_enabled = true; - } - - return 0; -} - -void resctrl_arch_mbm_cntr_assign_configure(void) -{ - struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - bool enable = true; - - mutex_lock(&rdtgroup_mutex); - - if (r->mon.mbm_cntr_assignable) { - if (!hw_res->mbm_cntr_assign_enabled) - hw_res->mbm_cntr_assign_enabled = true; - resctrl_abmc_set_one_amd(&enable); - } - - mutex_unlock(&rdtgroup_mutex); -} - -void resctrl_arch_mbm_cntr_assign_disable(void) -{ - struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - - lockdep_assert_held(&rdtgroup_mutex); - - if (hw_res->mbm_cntr_assign_enabled) { - _resctrl_abmc_enable(r, false); - hw_res->mbm_cntr_assign_enabled = false; - } -} - -bool resctrl_arch_get_mbm_cntr_assign_enable(void) -{ - struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - - return hw_res->mbm_cntr_assign_enabled; -} - -static void rdtgroup_abmc_cfg(void *info) -{ - u64 *msrval = info; - - wrmsrl(MSR_IA32_L3_QOS_ABMC_CFG, *msrval); -} - -/* - * Send an IPI to the domain to assign the counter id to RMID. - */ -int resctrl_arch_assign_cntr(void *dom, enum resctrl_event_id evtid, - u32 rmid, u32 cntr_id, u32 closid, bool assign) -{ - struct rdt_domain *d = dom; - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); - union l3_qos_abmc_cfg abmc_cfg = { 0 }; - struct arch_mbm_state *arch_mbm; - - abmc_cfg.split.cfg_en = 1; - abmc_cfg.split.cntr_en = assign ? 1 : 0; - abmc_cfg.split.cntr_id = cntr_id; - abmc_cfg.split.bw_src = rmid; - - /* Update the event configuration from the domain */ - if (evtid == QOS_L3_MBM_TOTAL_EVENT_ID) { - abmc_cfg.split.bw_type = hw_dom->mbm_total_cfg; - arch_mbm = &hw_dom->arch_mbm_total[rmid]; - } else { - abmc_cfg.split.bw_type = hw_dom->mbm_local_cfg; - arch_mbm = &hw_dom->arch_mbm_local[rmid]; - } - - smp_call_function_any(&d->cpu_mask, rdtgroup_abmc_cfg, &abmc_cfg, 1); - - /* - * Reset the architectural state so that reading of hardware - * counter is not considered as an overflow in next update. - */ - if (arch_mbm) - memset(arch_mbm, 0, sizeof(struct arch_mbm_state)); - - return 0; -} - -static int reset_all_ctrls(struct rdt_resource *r) -{ - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - struct rdt_hw_domain *hw_dom; + struct rdt_hw_ctrl_domain *hw_dom; struct msr_param msr_param; - cpumask_var_t cpu_mask; - struct rdt_domain *d; + struct rdt_ctrl_domain *d; int i; /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL)) - return -ENOMEM; - msr_param.res = r; msr_param.low = 0; msr_param.high = hw_res->num_closid; /* * Disable resource control for this resource by setting all - * CBMs in all domains to the maximum mask value. Pick one CPU + * CBMs in all ctrl_domains to the maximum mask value. Pick one CPU * from each domain to update the MSRs below. */ - list_for_each_entry(d, &r->domains, list) { - hw_dom = resctrl_to_arch_dom(d); - cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask); + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + hw_dom = resctrl_to_arch_ctrl_dom(d); for (i = 0; i < hw_res->num_closid; i++) - hw_dom->ctrl_val[i] = r->default_ctrl; - } - - /* Update CBM on all the CPUs in cpu_mask */ - on_each_cpu_mask(cpu_mask, rdt_ctrl_update, &msr_param, 1); - - free_cpumask_var(cpu_mask); - - return 0; -} - -void resctrl_arch_reset_resources(void) -{ - struct rdt_resource *r; - - for_each_capable_rdt_resource(r) - reset_all_ctrls(r); -} - -u32 resctrl_arch_event_config_get(void *dom, enum resctrl_event_id eventid) -{ - struct rdt_domain *d = dom; - struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); - - switch (eventid) { - case QOS_L3_OCCUP_EVENT_ID: - case QOS_MC_MBM_BPS_EVENT_ID: - break; - case QOS_L3_MBM_TOTAL_EVENT_ID: - return hw_dom->mbm_total_cfg; - case QOS_L3_MBM_LOCAL_EVENT_ID: - return hw_dom->mbm_local_cfg; - } - - /* Never expect to get here */ - WARN_ON_ONCE(1); - - return INVALID_CONFIG_VALUE; -} - -void resctrl_arch_event_config_set(void *info) -{ - struct resctrl_mon_config_info *mon_info = info; - struct rdt_hw_domain *hw_dom; - unsigned int index; - - index = mon_event_config_index_get(mon_info->evtid); - if (index == INVALID_CONFIG_INDEX) - return; - - wrmsr(MSR_IA32_EVT_CFG_BASE + index, mon_info->mon_config, 0); - - hw_dom = resctrl_to_arch_dom(mon_info->d); - - switch (mon_info->evtid) { - case QOS_L3_OCCUP_EVENT_ID: - case QOS_MC_MBM_BPS_EVENT_ID: - break; - case QOS_L3_MBM_TOTAL_EVENT_ID: - hw_dom->mbm_total_cfg = mon_info->mon_config; - break; - case QOS_L3_MBM_LOCAL_EVENT_ID: - hw_dom->mbm_local_cfg = mon_info->mon_config; - break; + hw_dom->ctrl_val[i] = resctrl_get_default_ctrl(r); + msr_param.dom = d; + smp_call_function_any(&d->hdr.cpu_mask, rdt_ctrl_update, &msr_param, 1); } - mon_info->err = 0; + return; } diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 0773ce13715fe5fffbe45f88040a4c37f68bf556..e1d8419f6e0878f2057dcc6ba35a7f4d4caf857f 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -52,11 +52,9 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_BMEC, CPUID_EBX, 3, 0x80000020, 0 }, { X86_FEATURE_TSA_SQ_NO, CPUID_ECX, 1, 0x80000021, 0 }, { X86_FEATURE_TSA_L1_NO, CPUID_ECX, 2, 0x80000021, 0 }, - { X86_FEATURE_ABMC, CPUID_EBX, 5, 0x80000020, 0 }, { X86_FEATURE_AMD_WORKLOAD_CLASS, CPUID_EAX, 22, 0x80000021, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, - { X86_FEATURE_SDCIAE, CPUID_EBX, 6, 0x80000020, 0 }, { X86_FEATURE_AMD_LBR_PMC_FREEZE, CPUID_EAX, 2, 0x80000022, 0 }, { 0, 0, 0, 0, 0 } }; diff --git a/drivers/acpi/arm64/Kconfig b/drivers/acpi/arm64/Kconfig index f2fd79f22e7d836b07401eb055725b2632ba624c..b3ed6212244c1e5405008355b7d0878252564251 100644 --- a/drivers/acpi/arm64/Kconfig +++ b/drivers/acpi/arm64/Kconfig @@ -21,6 +21,3 @@ config ACPI_AGDI config ACPI_APMT bool - -config ACPI_MPAM - bool diff --git a/drivers/acpi/arm64/Makefile b/drivers/acpi/arm64/Makefile index d8bd6066a4c67545ad0e75eb68ef27170b0d810e..726944648c9bcefa5d0462d855e72556513c15d9 100644 --- a/drivers/acpi/arm64/Makefile +++ b/drivers/acpi/arm64/Makefile @@ -4,6 +4,5 @@ obj-$(CONFIG_ACPI_IORT) += iort.o obj-$(CONFIG_ACPI_GTDT) += gtdt.o obj-$(CONFIG_ACPI_APMT) += apmt.o obj-$(CONFIG_ARM_AMBA) += amba.o -obj-$(CONFIG_ACPI_MPAM) += mpam.o obj-y += dma.o init.o obj-y += thermal_cpufreq.o diff --git a/drivers/acpi/arm64/mpam.c b/drivers/acpi/arm64/mpam.c deleted file mode 100644 index 153ef041abf0ca1e037b91dde21b86a62ba903ba..0000000000000000000000000000000000000000 --- a/drivers/acpi/arm64/mpam.c +++ /dev/null @@ -1,451 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2022 Arm Ltd. - -/* Parse the MPAM ACPI table feeding the discovered nodes into the driver */ - -#define pr_fmt(fmt) "ACPI MPAM: " fmt - -#include -#include -#include -#include -#include - -#include - -#include - -/* Flags for acpi_table_mpam_msc.*_interrupt_flags */ -#define ACPI_MPAM_MSC_IRQ_MODE_EDGE 1 -#define ACPI_MPAM_MSC_IRQ_TYPE_MASK (3<<1) -#define ACPI_MPAM_MSC_IRQ_TYPE_WIRED 0 -#define ACPI_MPAM_MSC_IRQ_AFFINITY_PROCESSOR_CONTAINER (1<<3) -#define ACPI_MPAM_MSC_IRQ_AFFINITY_VALID (1<<4) - -int ddrc_freq; - -/* Use OEM info in MPAM ACPI table to distinguish different machine types */ -struct acpi_mpam_machine_oem_info { - enum mpam_machine_type type; - char signature[ACPI_NAMESEG_SIZE + 1]; - u8 revision; - char oem_id[ACPI_OEM_ID_SIZE + 1]; - char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; - u32 oem_revision; -}; - -static struct acpi_mpam_machine_oem_info acpi_mpam_machines[MPAM_NUM_MACHINE_TYPES] = { - [MPAM_YITIAN710] = { - .signature = "YMPM", - .revision = 0, - .oem_id = "PTG ", - .oem_table_id = "PTG01 ", - .oem_revision = 0, - }, -}; - -static bool frob_irq(struct platform_device *pdev, int intid, u32 flags, - int *irq, u32 processor_container_uid) -{ - int sense; - - if (!intid) - return false; - - /* 0 in this field indicates a wired interrupt */ - if (flags & ACPI_MPAM_MSC_IRQ_TYPE_MASK) - return false; - - if (flags & ACPI_MPAM_MSC_IRQ_MODE_EDGE) - sense = ACPI_EDGE_SENSITIVE; - else - sense = ACPI_LEVEL_SENSITIVE; - - /* - * If the GSI is in the GIC's PPI range, try and create a partitioned - * percpu interrupt. - */ - if (16 <= intid && intid < 32 && processor_container_uid != ~0) { - pr_err_once("Partitioned interrupts not supported\n"); - return false; - } else { - *irq = acpi_register_gsi(&pdev->dev, intid, sense, - ACPI_ACTIVE_HIGH); - } - if (*irq <= 0) { - pr_err_once("Failed to register interrupt 0x%x with ACPI\n", - intid); - return false; - } - - return true; -} - -static void acpi_mpam_parse_irqs(struct platform_device *pdev, - struct acpi_mpam_msc_node *tbl_msc, - struct resource *res, int *res_idx) -{ - u32 flags, aff = ~0; - int irq; - - flags = tbl_msc->overflow_interrupt_flags; - if (flags & ACPI_MPAM_MSC_IRQ_AFFINITY_VALID && - flags & ACPI_MPAM_MSC_IRQ_AFFINITY_PROCESSOR_CONTAINER) - aff = tbl_msc->overflow_interrupt_affinity; - if (frob_irq(pdev, tbl_msc->overflow_interrupt, flags, &irq, aff)) { - res[*res_idx].start = irq; - res[*res_idx].end = irq; - res[*res_idx].flags = IORESOURCE_IRQ; - res[*res_idx].name = "overflow"; - - (*res_idx)++; - } - - flags = tbl_msc->error_interrupt_flags; - if (flags & ACPI_MPAM_MSC_IRQ_AFFINITY_VALID && - flags & ACPI_MPAM_MSC_IRQ_AFFINITY_PROCESSOR_CONTAINER) - aff = tbl_msc->error_interrupt_affinity; - else - aff = ~0; - if (frob_irq(pdev, tbl_msc->error_interrupt, flags, &irq, aff)) { - res[*res_idx].start = irq; - res[*res_idx].end = irq; - res[*res_idx].flags = IORESOURCE_IRQ; - res[*res_idx].name = "error"; - - (*res_idx)++; - } -} - -static int acpi_mpam_parse_resource(struct mpam_msc *msc, - struct acpi_mpam_resource_node *res) -{ - u32 cache_id; - int level; - - switch (res->locator_type) { - case ACPI_MPAM_LOCATION_TYPE_PROCESSOR_CACHE: - cache_id = res->locator.cache_locator.cache_reference; - if (mpam_current_machine == MPAM_YITIAN710) { - /* - * YITIAN710's BIOS doesn't support find level from - * cache id. Since it only supports L3 cache, use a - * fixed value, 3. - */ - level = 3; - } else { - level = find_acpi_cache_level_from_id(cache_id); - if (level < 0) { - pr_err_once("Bad level for cache with id %u\n", cache_id); - return level; - } - } - return mpam_ris_create(msc, res->ris_index, MPAM_CLASS_CACHE, - level, cache_id); - case ACPI_MPAM_LOCATION_TYPE_MEMORY: - if (mpam_current_machine == MPAM_YITIAN710) - ddrc_freq = res->locator.memory_locator.reserved; - return mpam_ris_create(msc, res->ris_index, MPAM_CLASS_MEMORY, - 255, res->locator.memory_locator.proximity_domain); - default: - /* These get discovered later and treated as unknown */ - return 0; - } -} - -int acpi_mpam_parse_resources(struct mpam_msc *msc, - struct acpi_mpam_msc_node *tbl_msc) -{ - int i, err; - struct acpi_mpam_resource_node *resources; - - resources = (struct acpi_mpam_resource_node *)(tbl_msc + 1); - for (i = 0; i < tbl_msc->num_resouce_nodes; i++) { - err = acpi_mpam_parse_resource(msc, &resources[i]); - if (err) - return err; - } - - return 0; -} - -static bool __init parse_msc_pm_link(struct acpi_mpam_msc_node *tbl_msc, - struct platform_device *pdev, - u32 *acpi_id) -{ - bool acpi_id_valid = false; - struct acpi_device *buddy; - char hid[16], uid[16]; - int err; - - memset(&hid, 0, sizeof(hid)); - memcpy(hid, &tbl_msc->hardware_id_linked_device, - sizeof(tbl_msc->hardware_id_linked_device)); - - if (!strcmp(hid, ACPI_PROCESSOR_CONTAINER_HID)) { - *acpi_id = tbl_msc->instance_id_linked_device; - acpi_id_valid = true; - } - - err = snprintf(uid, sizeof(uid), "%u", - tbl_msc->instance_id_linked_device); - if (err < 0 || err >= sizeof(uid)) - return acpi_id_valid; - - buddy = acpi_dev_get_first_match_dev(hid, uid, -1); - if (buddy) { - device_link_add(&pdev->dev, &buddy->dev, DL_FLAG_STATELESS); - } - - return acpi_id_valid; -} - -static int decode_interface_type(struct acpi_mpam_msc_node *tbl_msc, - enum mpam_msc_iface *iface) -{ - switch (tbl_msc->interface_type){ - case 0: - *iface = MPAM_IFACE_MMIO; - return 0; - case 1: - *iface = MPAM_IFACE_PCC; - return 0; - default: - return -EINVAL; - } -} - -static int __init _parse_table(struct acpi_table_header *table) -{ - char *table_end, *table_offset = (char *)(table + 1); - struct property_entry props[4]; /* needs a sentinel */ - struct acpi_mpam_msc_node *tbl_msc; - int next_res, next_prop, err = 0; - struct acpi_device *companion; - struct platform_device *pdev; - enum mpam_msc_iface iface; - struct resource res[3]; - char uid[16]; - u32 acpi_id; - int msc_num = 0; - - table_end = (char *)table + table->length; - - while (table_offset < table_end) { - tbl_msc = (struct acpi_mpam_msc_node *)table_offset; - table_offset += tbl_msc->length; - - /* - * If any of the reserved fields are set, make no attempt to - * parse the msc structure. This will prevent the driver from - * probing all the MSC, meaning it can't discover the system - * wide supported partid and pmg ranges. This avoids whatever - * this MSC is truncating the partids and creating a screaming - * error interrupt. - */ - if (tbl_msc->reserved || tbl_msc->reserved1 || tbl_msc->reserved2) - continue; - - if (decode_interface_type(tbl_msc, &iface)) - continue; - - next_res = 0; - next_prop = 0; - memset(res, 0, sizeof(res)); - memset(props, 0, sizeof(props)); - - /* - * Use an extra msc_num instead of msc->identifier, since MSC - * nodes with different types in MPAM ACPI table may have the - * same id value. - */ - pdev = platform_device_alloc("mpam_msc", msc_num++); - if (IS_ERR(pdev)) { - err = PTR_ERR(pdev); - break; - } - - if (tbl_msc->length < sizeof(*tbl_msc)) { - err = -EINVAL; - break; - } - - /* Some power management is described in the namespace: */ - err = snprintf(uid, sizeof(uid), "%u", tbl_msc->identifier); - if (err > 0 && err < sizeof(uid)) { - companion = acpi_dev_get_first_match_dev("ARMHAA5C", uid, -1); - if (companion) - ACPI_COMPANION_SET(&pdev->dev, companion); - } - - if (iface == MPAM_IFACE_MMIO) { - res[next_res].name = "MPAM:MSC"; - res[next_res].start = tbl_msc->base_address; - res[next_res].end = tbl_msc->base_address + tbl_msc->mmio_size - 1; - res[next_res].flags = IORESOURCE_MEM; - next_res++; - } else if (iface == MPAM_IFACE_PCC) { - props[next_prop++] = PROPERTY_ENTRY_U32("pcc-channel", - tbl_msc->base_address); - next_prop++; - } - - acpi_mpam_parse_irqs(pdev, tbl_msc, res, &next_res); - err = platform_device_add_resources(pdev, res, next_res); - if (err) - break; - - props[next_prop++] = PROPERTY_ENTRY_U32("arm,not-ready-us", - tbl_msc->max_nrdy_usec); - - /* - * The MSC's CPU affinity is described via its linked power - * management device, but only if it points at a Processor or - * Processor Container. - */ - if (parse_msc_pm_link(tbl_msc, pdev, &acpi_id)) { - props[next_prop++] = PROPERTY_ENTRY_U32("cpu_affinity", - acpi_id); - } - - err = device_create_managed_software_node(&pdev->dev, props, - NULL); - if (err) - break; - - /* Come back later if you want the RIS too */ - err = platform_device_add_data(pdev, tbl_msc, tbl_msc->length); - if (err) - break; - - platform_device_add(pdev); - } - - if (err) - platform_device_put(pdev); - - return err; -} - -static struct acpi_table_header *get_table(void) -{ - struct acpi_table_header *table; - enum mpam_machine_type mtype; - acpi_status status; - - if (acpi_disabled || !mpam_cpus_have_feature()) - return NULL; - - mtype = acpi_mpam_get_machine_type(); - - if (mtype != MPAM_DEFAULT_MACHINE) - status = acpi_get_table(acpi_mpam_machines[mtype].signature, 0, &table); - else - status = acpi_get_table(ACPI_SIG_MPAM, 0, &table); - if (ACPI_FAILURE(status)) - return NULL; - - if (mtype == MPAM_DEFAULT_MACHINE && table->revision != 1) - return NULL; - - /* - * Kunpeng's MPAM ACPI adopts an older version of MPAM ACPI, so - * this MPAM ACPI driver is not suitable for Kunpeng platform. - * Skip it. - */ - if (!strncmp(table->oem_id, "HISI", 4)) { - acpi_put_table(table); - return NULL; - } - - return table; -} - - - -static int __init acpi_mpam_parse(void) -{ - struct acpi_table_header *mpam; - int err; - - mpam = get_table(); - if (!mpam) - return 0; - - err = _parse_table(mpam); - acpi_put_table(mpam); - - return err; -} - -static int _count_msc(struct acpi_table_header *table) -{ - char *table_end, *table_offset = (char *)(table + 1); - struct acpi_mpam_msc_node *tbl_msc; - int ret = 0; - - tbl_msc = (struct acpi_mpam_msc_node *)table_offset; - table_end = (char *)table + table->length; - - while (table_offset < table_end) { - if (tbl_msc->length < sizeof(*tbl_msc)) - return -EINVAL; - - ret++; - - table_offset += tbl_msc->length; - tbl_msc = (struct acpi_mpam_msc_node *)table_offset; - } - - return ret; -} - - -int acpi_mpam_count_msc(void) -{ - struct acpi_table_header *mpam; - int ret; - - mpam = get_table(); - if (!mpam) - return 0; - - ret = _count_msc(mpam); - acpi_put_table(mpam); - - return ret; -} - -enum mpam_machine_type acpi_mpam_get_machine_type(void) -{ - struct acpi_table_header *table; - enum mpam_machine_type ret; - acpi_status status; - int i; - - ret = MPAM_DEFAULT_MACHINE; - - for (i = MPAM_DEFAULT_MACHINE + 1; i < MPAM_NUM_MACHINE_TYPES; i++) { - status = acpi_get_table(acpi_mpam_machines[i].signature, 0, &table); - if (ACPI_FAILURE(status)) - continue; - - if (!memcmp(acpi_mpam_machines[i].oem_id, table->oem_id, ACPI_OEM_ID_SIZE) && - !memcmp(acpi_mpam_machines[i].oem_table_id, table->oem_table_id, - ACPI_OEM_TABLE_ID_SIZE) && - acpi_mpam_machines[i].oem_revision == table->oem_revision) { - ret = i; - } - - acpi_put_table(table); - } - - return ret; -} - -/* - * Call after ACPI devices have been created, which happens behind acpi_scan_init() - * called from subsys_initcall(). PCC requires the mailbox driver, which is - * initialised from postcore_initcall(). - */ -subsys_initcall_sync(acpi_mpam_parse); diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c index eea6ea5fcdcad0a09c1ed18f9a491b20151bcc69..54676e3d82dd598a76bfbec10440eae57483365a 100644 --- a/drivers/acpi/pptt.c +++ b/drivers/acpi/pptt.c @@ -21,8 +21,6 @@ #include #include -typedef int (*acpi_pptt_cpu_callback_t)(struct acpi_pptt_processor *, void *); - static struct acpi_subtable_header *fetch_pptt_subtable(struct acpi_table_header *table_hdr, u32 pptt_ref) { @@ -183,10 +181,9 @@ acpi_find_cache_level(struct acpi_table_header *table_hdr, * levels and split cache levels (data/instruction). * @table_hdr: Pointer to the head of the PPTT table * @cpu_node: processor node we wish to count caches for - * @levels: Number of levels if success. (*levels) should be initialized by - * the caller with the value to be used as the starting level. + * @levels: Number of levels if success. * @split_levels: Number of split cache levels (data/instruction) if - * success. Can be NULL. + * success. Can by NULL. * * Given a processor node containing a processing unit, walk into it and count * how many levels exist solely for it, and then walk up each level until we hit @@ -301,177 +298,6 @@ static struct acpi_pptt_processor *acpi_find_processor_node(struct acpi_table_he return NULL; } -/* - * acpi_pptt_find_cache_backwards() - Given a PPTT cache find a processor node - * that points to it. This lets us find a cacheinfo node by fw_token, but - * is totally broken as many processor node may point at the same PPTT - * cache indicating different instances of the cache. (e.g. all the L1 - * caches are the same shape, but they aren't the same cache). - * This only works if you cooked your PPTT table to look like this. - */ -struct acpi_pptt_processor * -acpi_pptt_find_cache_backwards(struct acpi_table_header *table_hdr, - struct acpi_pptt_cache *cache) -{ - struct acpi_pptt_processor *cpu_node; - struct acpi_subtable_header *entry; - struct acpi_subtable_header *res; - unsigned long table_end; - u32 proc_sz; - int i; - - table_end = (unsigned long)table_hdr + table_hdr->length; - entry = ACPI_ADD_PTR(struct acpi_subtable_header, table_hdr, - sizeof(struct acpi_table_pptt)); - proc_sz = sizeof(struct acpi_pptt_processor *); - - /* find the processor structure which points at with this cpuid */ - while ((unsigned long)entry + proc_sz < table_end) { - if (entry->length == 0) { - pr_warn("Invalid zero length subtable\n"); - break; - } - - cpu_node = (struct acpi_pptt_processor *)entry; - entry = ACPI_ADD_PTR(struct acpi_subtable_header, entry, - entry->length); - - if (cpu_node->header.type != ACPI_PPTT_TYPE_PROCESSOR) - continue; - - for (i = 0; i < cpu_node->number_of_priv_resources; i++) { - res = acpi_get_pptt_resource(table_hdr, cpu_node, i); - if (&cache->header == res) - return cpu_node; - } - } - - return NULL; -} - -/* parent_node points into the table, but the table isn't provided. */ -static void acpi_pptt_get_child_cpus(struct acpi_pptt_processor *parent_node, - cpumask_t *cpus) -{ - struct acpi_pptt_processor *cpu_node; - struct acpi_table_header *table_hdr; - acpi_status status; - u32 acpi_id; - int cpu; - - status = acpi_get_table(ACPI_SIG_PPTT, 0, &table_hdr); - if (ACPI_FAILURE(status)) - return; - - for_each_possible_cpu(cpu) { - acpi_id = get_acpi_id_for_cpu(cpu); - cpu_node = acpi_find_processor_node(table_hdr, acpi_id); - - while (cpu_node) { - if (cpu_node == parent_node) { - cpumask_set_cpu(cpu, cpus); - break; - } - cpu_node = fetch_pptt_node(table_hdr, cpu_node->parent); - } - } - - acpi_put_table(table_hdr); - - return; -} - -/** - * acpi_pptt_for_each_container() - Iterate over all processor containers - * - * Not all 'Processor' entries in the PPTT are either a CPU or a Processor - * Container, they may exist purely to describe a Private resource. CPUs - * have to be leaves, so a Processor Container is a non-leaf that has the - * 'ACPI Processor ID valid' flag set. - * - * Return: 0 for a complete walk, or the first non-zero value from the callback - * that stopped the walk. - */ -int acpi_pptt_for_each_container(acpi_pptt_cpu_callback_t callback, void *arg) -{ - struct acpi_pptt_processor *cpu_node; - struct acpi_table_header *table_hdr; - struct acpi_subtable_header *entry; - bool leaf_flag, has_leaf_flag = false; - unsigned long table_end; - acpi_status status; - u32 proc_sz; - int ret = 0; - - status = acpi_get_table(ACPI_SIG_PPTT, 0, &table_hdr); - if (ACPI_FAILURE(status)) - return 0; - - if (table_hdr->revision > 1) - has_leaf_flag = true; - - table_end = (unsigned long)table_hdr + table_hdr->length; - entry = ACPI_ADD_PTR(struct acpi_subtable_header, table_hdr, - sizeof(struct acpi_table_pptt)); - proc_sz = sizeof(struct acpi_pptt_processor); - while ((unsigned long)entry + proc_sz < table_end) { - cpu_node = (struct acpi_pptt_processor *)entry; - if (entry->type == ACPI_PPTT_TYPE_PROCESSOR && - cpu_node->flags & ACPI_PPTT_ACPI_PROCESSOR_ID_VALID) - { - leaf_flag = cpu_node->flags & ACPI_PPTT_ACPI_LEAF_NODE; - if ((has_leaf_flag && !leaf_flag) || - (!has_leaf_flag && !acpi_pptt_leaf_node(table_hdr, cpu_node))) - { - ret = callback(cpu_node, arg); - if (ret) - break; - } - } - entry = ACPI_ADD_PTR(struct acpi_subtable_header, entry, - entry->length); - } - - acpi_put_table(table_hdr); - - return ret; -} - -struct __cpus_from_container_arg { - u32 acpi_cpu_id; - cpumask_t *cpus; -}; - -static int __cpus_from_container(struct acpi_pptt_processor *container, void *arg) -{ - struct __cpus_from_container_arg *params = arg; - - if (container->acpi_processor_id == params->acpi_cpu_id) - acpi_pptt_get_child_cpus(container, params->cpus); - - return 0; -} - -/** - * acpi_pptt_get_cpus_from_container() - Populate a cpumask with all CPUs in a - * processor containers - * - * Find the specified Processor Container, and fill cpus with all the cpus - * below it. - * - * Return: 0 for a complete walk, or an error if the mask is incomplete. - */ -int acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id, cpumask_t *cpus) -{ - struct __cpus_from_container_arg params; - - params.acpi_cpu_id = acpi_cpu_id; - params.cpus = cpus; - - cpumask_clear(cpus); - return acpi_pptt_for_each_container(&__cpus_from_container, ¶ms); -} - static u8 acpi_cache_type(enum cache_type type) { switch (type) { @@ -991,215 +817,3 @@ int find_acpi_cpu_topology_hetero_id(unsigned int cpu) return find_acpi_cpu_topology_tag(cpu, PPTT_ABORT_PACKAGE, ACPI_PPTT_ACPI_IDENTICAL); } - - -/** - * find_acpi_cache_level_from_id() - Get the level of the specified cache - * @cache_id: The id field of the unified cache - * - * Determine the level relative to any CPU for the unified cache identified by - * cache_id. This allows the property to be found even if the CPUs are offline. - * - * The returned level can be used to group unified caches that are peers. - * - * The PPTT table must be rev 3 or later, - * - * If one CPUs L2 is shared with another as L3, this function will return - * and unpredictable value. - * - * Return: -ENOENT if the PPTT doesn't exist, or the cache cannot be found. - * Otherwise returns a value which represents the level of the specified cache. - */ -int find_acpi_cache_level_from_id(u32 cache_id) -{ - u32 acpi_cpu_id; - acpi_status status; - int level, cpu, num_levels; - struct acpi_pptt_cache *cache; - struct acpi_table_header *table; - struct acpi_pptt_cache_v1* cache_v1; - struct acpi_pptt_processor *cpu_node; - - status = acpi_get_table(ACPI_SIG_PPTT, 0, &table); - if (ACPI_FAILURE(status)) { - acpi_pptt_warn_missing(); - return -ENOENT; - } - - if (table->revision < 3) { - acpi_put_table(table); - return -ENOENT; - } - - /* - * If we found the cache first, we'd still need to walk from each CPU - * to find the level... - */ - for_each_possible_cpu(cpu) { - - num_levels = 0; - acpi_cpu_id = get_acpi_id_for_cpu(cpu); - cpu_node = acpi_find_processor_node(table, acpi_cpu_id); - if (!cpu_node) - break; - acpi_count_levels(table, cpu_node, &num_levels, NULL); - - /* Start at 1 for L1 */ - for (level = 1; level <= num_levels; level++) { - cache = acpi_find_cache_node(table, acpi_cpu_id, - ACPI_PPTT_CACHE_TYPE_UNIFIED, - level, &cpu_node); - if (!cache) - continue; - - cache_v1 = ACPI_ADD_PTR(struct acpi_pptt_cache_v1, - cache, - sizeof(struct acpi_pptt_cache)); - - if (cache->flags & ACPI_PPTT_CACHE_ID_VALID && - cache_v1->cache_id == cache_id) { - acpi_put_table(table); - return level; - } - } - } - - acpi_put_table(table); - return -ENOENT; -} - -/** - * acpi_pptt_get_cpumask_from_cache_id() - Get the cpus associated with the - * specified cache - * @cache_id: The id field of the unified cache - * @cpus: Where to buidl the cpumask - * - * Determine which CPUs are below this cache in the PPTT. This allows the property - * to be found even if the CPUs are offline. - * - * The PPTT table must be rev 3 or later, - * - * Return: -ENOENT if the PPTT doesn't exist, or the cache cannot be found. - * Otherwise returns 0 and sets the cpus in the provided cpumask. - */ -int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, cpumask_t *cpus) -{ - u32 acpi_cpu_id; - acpi_status status; - int level, cpu, num_levels; - struct acpi_pptt_cache *cache; - struct acpi_table_header *table; - struct acpi_pptt_cache_v1* cache_v1; - struct acpi_pptt_processor *cpu_node; - - cpumask_clear(cpus); - - status = acpi_get_table(ACPI_SIG_PPTT, 0, &table); - if (ACPI_FAILURE(status)) { - acpi_pptt_warn_missing(); - return -ENOENT; - } - - if (table->revision < 3) { - acpi_put_table(table); - return -ENOENT; - } - - /* - * If we found the cache first, we'd still need to walk from each cpu. - */ - for_each_possible_cpu(cpu) { - - num_levels = 0; - acpi_cpu_id = get_acpi_id_for_cpu(cpu); - cpu_node = acpi_find_processor_node(table, acpi_cpu_id); - if (!cpu_node) - break; - acpi_count_levels(table, cpu_node, &num_levels, NULL); - - /* Start at 1 for L1 */ - for (level = 1; level <= num_levels; level++) { - cache = acpi_find_cache_node(table, acpi_cpu_id, - ACPI_PPTT_CACHE_TYPE_UNIFIED, - level, &cpu_node); - if (!cache) - continue; - - cache_v1 = ACPI_ADD_PTR(struct acpi_pptt_cache_v1, - cache, - sizeof(struct acpi_pptt_cache)); - - if (cache->flags & ACPI_PPTT_CACHE_ID_VALID && - cache_v1->cache_id == cache_id) { - cpumask_set_cpu(cpu, cpus); - } - } - } - - acpi_put_table(table); - return 0; -} - -/** - * acpi_pptt_get_cpumask_from_cache_id_and_level() - Get the cpus associated with the - * cache specified by id and level - * @cache_id: The id field of the unified cache - * @cache_level: The level of the unified cache - * @cpus: Where to buidl the cpumask - * - * Determine which CPUs are below this cache in the PPTT. This allows the property - * to be found even if the CPUs are offline. - * - * The PPTT table must be rev 3 or later, - * - * Return: -ENOENT if the PPTT doesn't exist, or the cache cannot be found. - * Otherwise returns 0 and sets the cpus in the provided cpumask. - */ -int acpi_pptt_get_cpumask_from_cache_id_and_level(u32 cache_id, u32 cache_level, - cpumask_t *cpus) -{ - u32 acpi_cpu_id; - acpi_status status; - int cpu; - struct acpi_table_header *table; - struct acpi_pptt_cache *cache_node; - struct acpi_pptt_processor *cpu_node; - - cpumask_clear(cpus); - - status = acpi_get_table(ACPI_SIG_PPTT, 0, &table); - if (ACPI_FAILURE(status)) { - acpi_pptt_warn_missing(); - return -ENOENT; - } - - /* - * FIXME: Since this function does not actually use the cache id in the - * PPTT table, we downgrade the revision requirement. - */ - if (table->revision < 2) { - acpi_put_table(table); - return -ENOENT; - } - - for_each_possible_cpu(cpu) { - acpi_cpu_id = get_acpi_id_for_cpu(cpu); - cpu_node = acpi_find_processor_node(table, acpi_cpu_id); - if (!cpu_node) - continue; - - cache_node = acpi_find_cache_node(table, acpi_cpu_id, - ACPI_PPTT_CACHE_TYPE_UNIFIED, - cache_level, &cpu_node); - - if (!cache_node) - continue; - - cpu_node = acpi_pptt_find_cache_backwards(table, cache_node); - if (cpu_node->acpi_processor_id == cache_id) - cpumask_set_cpu(cpu, cpus); - } - - acpi_put_table(table); - return 0; -} diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 8961e4101a1d044cef058be6d73e35332624361a..6b4c292f989d2f0753fd699132402e6b0a243467 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -431,7 +431,7 @@ static const char table_sigs[][ACPI_NAMESEG_SIZE] __initconst = { ACPI_SIG_PSDT, ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT, ACPI_SIG_IORT, ACPI_SIG_NFIT, ACPI_SIG_HMAT, ACPI_SIG_PPTT, ACPI_SIG_NHLT, ACPI_SIG_AEST, ACPI_SIG_CEDT, ACPI_SIG_AGDI, - ACPI_SIG_NBFT, ACPI_SIG_MPAM }; + ACPI_SIG_NBFT }; #define ACPI_HEADER_SIZE sizeof(struct acpi_table_header) diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index 24200cb949ccd57a86bd4a3714c716c92d58de30..0fbb1575e297ab90bcbb4ced9858bedd257abf07 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -183,38 +183,6 @@ static bool cache_node_is_unified(struct cacheinfo *this_leaf, return of_property_read_bool(np, "cache-unified"); } -unsigned long cache_of_get_id(struct device_node *np) -{ - struct device_node *cpu; - unsigned long min_id = ~0UL; - - for_each_of_cpu_node(cpu) { - struct device_node *cache_node = cpu; - u64 id = of_get_cpu_hwid(cache_node, 0); - - while ((cache_node = of_find_next_cache_node(cache_node))) { - if ((cache_node == np) && (id < min_id)) { - min_id = id; - of_node_put(cache_node); - break; - } - of_node_put(cache_node); - } - } - - return min_id; -} - -static void cache_of_set_id(struct cacheinfo *this_leaf, struct device_node *np) -{ - unsigned long id = cache_of_get_id(np); - - if (id != ~0UL) { - this_leaf->id = id; - this_leaf->attributes |= CACHE_ID; - } -} - static void cache_of_set_props(struct cacheinfo *this_leaf, struct device_node *np) { @@ -230,7 +198,6 @@ static void cache_of_set_props(struct cacheinfo *this_leaf, cache_get_line_size(this_leaf, np); cache_nr_sets(this_leaf, np); cache_associativity(this_leaf); - cache_of_set_id(this_leaf, np); } static int cache_setup_of_node(unsigned int cpu) @@ -655,19 +622,13 @@ static ssize_t file_name##_show(struct device *dev, \ return sysfs_emit(buf, "%u\n", this_leaf->object); \ } +show_one(id, id); show_one(level, level); show_one(coherency_line_size, coherency_line_size); show_one(number_of_sets, number_of_sets); show_one(physical_line_partition, physical_line_partition); show_one(ways_of_associativity, ways_of_associativity); -static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - struct cacheinfo *this_leaf = dev_get_drvdata(dev); - - return sysfs_emit(buf, "%lu\n", this_leaf->id); -} - static ssize_t size_show(struct device *dev, struct device_attribute *attr, char *buf) { diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 2f022d1fb052b039fd31ae97fedb7ff8795901b5..77aa37de59880f33eeb35f43747333dd5a34e2d5 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -2435,7 +2435,6 @@ static int arm_cmn_probe(struct platform_device *pdev) struct arm_cmn *cmn; const char *name; static atomic_t id; - struct resource *cfg; int err, rootnode, this_id; cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL); @@ -2451,16 +2450,7 @@ static int arm_cmn_probe(struct platform_device *pdev) rootnode = arm_cmn600_acpi_probe(pdev, cmn); } else { rootnode = 0; - - /* - * Avoid registering resources as the PMUs registers are - * scattered through CMN, and may appear either side of - * registers for other 'devices'. (e.g. the MPAM MSC controls). - */ - cfg = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!cfg) - return -EINVAL; - cmn->base = devm_ioremap(&pdev->dev, cfg->start, resource_size(cfg)); + cmn->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(cmn->base)) return PTR_ERR(cmn->base); if (cmn->part == PART_CMN600) diff --git a/drivers/platform/Kconfig b/drivers/platform/Kconfig index f26534a4a83b5aedb85c616f90db739a0ceaba4a..868b20361769c37048ef58392d9aae43abcaf021 100644 --- a/drivers/platform/Kconfig +++ b/drivers/platform/Kconfig @@ -9,8 +9,6 @@ source "drivers/platform/chrome/Kconfig" source "drivers/platform/mellanox/Kconfig" -source "drivers/platform/mpam/Kconfig" - source "drivers/platform/olpc/Kconfig" source "drivers/platform/surface/Kconfig" diff --git a/drivers/platform/Makefile b/drivers/platform/Makefile index 54ee16e4e4d8a96e7795dfe152827693b862ad56..8296d4c41eb7706133d909b215e6f059996256f1 100644 --- a/drivers/platform/Makefile +++ b/drivers/platform/Makefile @@ -12,4 +12,3 @@ obj-$(CONFIG_OLPC_EC) += olpc/ obj-$(CONFIG_GOLDFISH) += goldfish/ obj-$(CONFIG_CHROME_PLATFORMS) += chrome/ obj-$(CONFIG_SURFACE_PLATFORMS) += surface/ -obj-$(CONFIG_ARM_CPU_RESCTRL) += mpam/ diff --git a/drivers/platform/mpam/Kconfig b/drivers/platform/mpam/Kconfig deleted file mode 100644 index 75f5b2454fbe45b9531e6a680ecdc55df166200b..0000000000000000000000000000000000000000 --- a/drivers/platform/mpam/Kconfig +++ /dev/null @@ -1,8 +0,0 @@ -# Confusingly, this is everything but the CPU bits of MPAM. CPU here means -# CPU resources, not containers or cgroups etc. -config ARM_CPU_RESCTRL - bool - default y - depends on ARM64 && ARCH_HAS_CPU_RESCTRL - depends on MISC_FILESYSTEMS - select RESCTRL_RMID_DEPENDS_ON_CLOSID diff --git a/drivers/platform/mpam/Makefile b/drivers/platform/mpam/Makefile deleted file mode 100644 index 37693be531c34151078e6e6055d0bb4eea9099bf..0000000000000000000000000000000000000000 --- a/drivers/platform/mpam/Makefile +++ /dev/null @@ -1 +0,0 @@ -obj-$(CONFIG_ARM_CPU_RESCTRL) += mpam_devices.o mpam_resctrl.o diff --git a/drivers/platform/mpam/mpam_devices.c b/drivers/platform/mpam/mpam_devices.c deleted file mode 100644 index c7dfc1e17ed69f1b94381a23986ed75431335abd..0000000000000000000000000000000000000000 --- a/drivers/platform/mpam/mpam_devices.c +++ /dev/null @@ -1,2529 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2022 Arm Ltd. - -#define pr_fmt(fmt) "mpam: " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include "mpam_internal.h" - -extern int ddrc_freq; - -/* - * mpam_list_lock protects the SRCU lists when writing. Once the - * mpam_enabled key is enabled these lists are read-only, - * unless the error interrupt disables the driver. - */ -static DEFINE_MUTEX(mpam_list_lock); -static LIST_HEAD(mpam_all_msc); - -struct srcu_struct mpam_srcu; - -enum mpam_machine_type mpam_current_machine; - -/* MPAM isn't available until all the MSC have been probed. */ -static u32 mpam_num_msc; - -static int mpam_cpuhp_state; -static DEFINE_MUTEX(mpam_cpuhp_state_lock); - -/* - * The smallest common values for any CPU or MSC in the system. - * Generating traffic outside this range will result in screaming interrupts. - */ -u16 mpam_partid_max; -u8 mpam_pmg_max; -static bool partid_max_init, partid_max_published; -static DEFINE_SPINLOCK(partid_max_lock); - -/* - * mpam is enabled once all devices have been probed from CPU online callbacks, - * scheduled via this work_struct. If access to an MSC depends on a CPU that - * was not brought online at boot, this can happen surprisingly late. - */ -static DECLARE_WORK(mpam_enable_work, &mpam_enable); - -/* - * All mpam error interrupts indicate a software bug. On receipt, disable the - * driver. - */ -static DECLARE_WORK(mpam_broken_work, &mpam_disable); - -/* - * An MSC is a container for resources, each identified by their RIS index. - * Components are a group of RIS that control the same thing. - * Classes are the set components of the same type. - * - * e.g. The set of RIS that make up the L2 are a component. These are sometimes - * termed slices. They should be configured as if they were one MSC. - * - * e.g. The SoC probably has more than one L2, each attached to a distinct set - * of CPUs. All the L2 components are grouped as a class. - * - * When creating an MSC, struct mpam_msc is added to the all mpam_all_msc list, - * then linked via struct mpam_ris to a component and a class. - * The same MSC may exist under different class->component paths, but the RIS - * index will be unique. - */ -LIST_HEAD(mpam_classes); - -static u32 __mpam_read_reg(struct mpam_msc *msc, u16 reg) -{ - WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz); - WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); - - return readl_relaxed(msc->mapped_hwpage + reg); -} - -static void __mpam_write_reg(struct mpam_msc *msc, u16 reg, u32 val) -{ - WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz); - WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); - - writel_relaxed(val, msc->mapped_hwpage + reg); -} - -#define mpam_read_partsel_reg(msc, reg) \ -({ \ - u32 ____ret; \ - \ - lockdep_assert_held_once(&msc->part_sel_lock); \ - ____ret = __mpam_read_reg(msc, MPAMF_##reg); \ - \ - ____ret; \ -}) - -#define mpam_write_partsel_reg(msc, reg, val) \ -({ \ - lockdep_assert_held_once(&msc->part_sel_lock); \ - __mpam_write_reg(msc, MPAMCFG_##reg, val); \ -}) - -#define mpam_read_monsel_reg(msc, reg) \ -({ \ - u32 ____ret; \ - \ - lockdep_assert_held_once(&msc->mon_sel_lock); \ - ____ret = __mpam_read_reg(msc, MSMON_##reg); \ - \ - ____ret; \ -}) - -#define mpam_write_monsel_reg(msc, reg, val) \ -({ \ - lockdep_assert_held_once(&msc->mon_sel_lock); \ - __mpam_write_reg(msc, MSMON_##reg, val); \ -}) - -static u64 mpam_msc_read_idr(struct mpam_msc *msc) -{ - u64 idr_high = 0, idr_low; - - lockdep_assert_held(&msc->part_sel_lock); - - idr_low = mpam_read_partsel_reg(msc, IDR); - if (FIELD_GET(MPAMF_IDR_HAS_EXT, idr_low)) - idr_high = mpam_read_partsel_reg(msc, IDR + 4); - - return (idr_high << 32) | idr_low; -} - -static void mpam_msc_zero_esr(struct mpam_msc *msc) -{ - writel_relaxed(0, msc->mapped_hwpage + MPAMF_ESR); - if (msc->has_extd_esr) - writel_relaxed(0, msc->mapped_hwpage + MPAMF_ESR + 4); -} - -static u64 mpam_msc_read_esr(struct mpam_msc *msc) -{ - u64 esr_high = 0, esr_low; - - esr_low = readl_relaxed(msc->mapped_hwpage + MPAMF_ESR); - if (msc->has_extd_esr) - esr_high = readl_relaxed(msc->mapped_hwpage + MPAMF_ESR + 4); - - return (esr_high << 32) | esr_low; -} - -static void __mpam_part_sel(u8 ris_idx, u16 partid, struct mpam_msc *msc) -{ - u32 partsel; - - lockdep_assert_held(&msc->part_sel_lock); - - partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) | - FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, partid); - mpam_write_partsel_reg(msc, PART_SEL, partsel); -} - -int mpam_register_requestor(u16 partid_max, u8 pmg_max) -{ - int err = 0; - - spin_lock(&partid_max_lock); - if (!partid_max_init) { - mpam_partid_max = partid_max; - mpam_pmg_max = pmg_max; - partid_max_init = true; - } else if (!partid_max_published) { - mpam_partid_max = min(mpam_partid_max, partid_max); - mpam_pmg_max = min(mpam_pmg_max, pmg_max); - } else { - /* New requestors can't lower the values */ - if ((partid_max < mpam_partid_max) || (pmg_max < mpam_pmg_max)) - err = -EBUSY; - } - spin_unlock(&partid_max_lock); - - return err; -} -EXPORT_SYMBOL(mpam_register_requestor); - -static struct mpam_component * -mpam_component_alloc(struct mpam_class *class, int id, gfp_t gfp) -{ - struct mpam_component *comp; - - lockdep_assert_held(&mpam_list_lock); - - comp = kzalloc(sizeof(*comp), gfp); - if (!comp) - return ERR_PTR(-ENOMEM); - - comp->comp_id = id; - INIT_LIST_HEAD_RCU(&comp->ris); - /* affinity is updated when ris are added */ - INIT_LIST_HEAD_RCU(&comp->class_list); - comp->class = class; - - list_add_rcu(&comp->class_list, &class->components); - - return comp; -} - -static struct mpam_component * -mpam_component_get(struct mpam_class *class, int id, bool alloc, gfp_t gfp) -{ - struct mpam_component *comp; - - lockdep_assert_held(&mpam_list_lock); - - list_for_each_entry(comp, &class->components, class_list) { - if (comp->comp_id == id) - return comp; - } - - if (!alloc) - return ERR_PTR(-ENOENT); - - return mpam_component_alloc(class, id, gfp); -} - -static struct mpam_class * -mpam_class_alloc(u8 level_idx, enum mpam_class_types type, gfp_t gfp) -{ - struct mpam_class *class; - - lockdep_assert_held(&mpam_list_lock); - - class = kzalloc(sizeof(*class), gfp); - if (!class) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD_RCU(&class->components); - /* affinity is updated when ris are added */ - class->level = level_idx; - class->type = type; - INIT_LIST_HEAD_RCU(&class->classes_list); - ida_init(&class->ida_csu_mon); - ida_init(&class->ida_mbwu_mon); - - list_add_rcu(&class->classes_list, &mpam_classes); - - return class; -} - -static struct mpam_class * -mpam_class_get(u8 level_idx, enum mpam_class_types type, bool alloc, gfp_t gfp) -{ - bool found = false; - struct mpam_class *class; - - lockdep_assert_held(&mpam_list_lock); - - list_for_each_entry(class, &mpam_classes, classes_list) { - if (class->type == type && class->level == level_idx) { - found = true; - break; - } - } - - if (found) - return class; - - if (!alloc) - return ERR_PTR(-ENOENT); - - return mpam_class_alloc(level_idx, type, gfp); -} - -static void mpam_class_destroy(struct mpam_class *class) -{ - lockdep_assert_held(&mpam_list_lock); - - list_del_rcu(&class->classes_list); - synchronize_srcu(&mpam_srcu); - kfree(class); -} - -static void mpam_comp_destroy(struct mpam_component *comp) -{ - struct mpam_class *class = comp->class; - - lockdep_assert_held(&mpam_list_lock); - - list_del_rcu(&comp->class_list); - synchronize_srcu(&mpam_srcu); - kfree(comp); - - if (list_empty(&class->components)) - mpam_class_destroy(class); -} - -/* synchronise_srcu() before freeing ris */ -static void mpam_ris_destroy(struct mpam_msc_ris *ris) -{ - struct mpam_component *comp = ris->comp; - struct mpam_class *class = comp->class; - struct mpam_msc *msc = ris->msc; - - lockdep_assert_held(&mpam_list_lock); - lockdep_assert_preemption_enabled(); - - clear_bit(ris->ris_idx, msc->ris_idxs); - list_del_rcu(&ris->comp_list); - list_del_rcu(&ris->msc_list); - - cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity); - cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity); - - if (list_empty(&comp->ris)) - mpam_comp_destroy(comp); -} - -/* - * There are two ways of reaching a struct mpam_msc_ris. Via the - * class->component->ris, or via the msc. - * When destroying the msc, the other side needs unlinking and cleaning up too. - * synchronise_srcu() before freeing msc. - */ -static void mpam_msc_destroy(struct mpam_msc *msc) -{ - struct mpam_msc_ris *ris, *tmp; - - lockdep_assert_held(&mpam_list_lock); - lockdep_assert_preemption_enabled(); - - list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list) - mpam_ris_destroy(ris); -} - -/* - * The cacheinfo structures are only populated when CPUs are online. - * This helper walks the device tree to include offline CPUs too. - */ -static int get_cpumask_from_cache_id(u32 cache_id, u32 cache_level, - cpumask_t *affinity) -{ - int cpu, err; - u32 iter_level; - int iter_cache_id; - struct device_node *iter; - - if (!acpi_disabled) { - if (mpam_current_machine == MPAM_YITIAN710) - return acpi_pptt_get_cpumask_from_cache_id_and_level( - cache_id, cache_level, affinity); - return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity); - } - - for_each_possible_cpu(cpu) { - iter = of_get_cpu_node(cpu, NULL); - if (!iter) { - pr_err("Failed to find cpu%d device node\n", cpu); - return -ENOENT; - } - - while ((iter = of_find_next_cache_node(iter))) { - err = of_property_read_u32(iter, "cache-level", - &iter_level); - if (err || (iter_level != cache_level)) { - of_node_put(iter); - continue; - } - - /* - * get_cpu_cacheinfo_id() isn't ready until sometime - * during device_initcall(). Use cache_of_get_id(). - */ - iter_cache_id = cache_of_get_id(iter); - if (cache_id == ~0UL) { - of_node_put(iter); - continue; - } - - if (iter_cache_id == cache_id) - cpumask_set_cpu(cpu, affinity); - - of_node_put(iter); - } - } - - return 0; -} - - -/* - * cpumask_of_node() only knows about online CPUs. This can't tell us whether - * a class is represented on all possible CPUs. - */ -static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity) -{ - int cpu; - - for_each_possible_cpu(cpu) { - if (node_id == cpu_to_node(cpu)) - cpumask_set_cpu(cpu, affinity); - } -} - -static int get_cpumask_from_cache(struct device_node *cache, - cpumask_t *affinity) -{ - int err; - u32 cache_level; - int cache_id; - - err = of_property_read_u32(cache, "cache-level", &cache_level); - if (err) { - pr_err("Failed to read cache-level from cache node\n"); - return -ENOENT; - } - - cache_id = cache_of_get_id(cache); - if (cache_id == ~0UL) { - pr_err("Failed to calculate cache-id from cache node\n"); - return -ENOENT; - } - - return get_cpumask_from_cache_id(cache_id, cache_level, affinity); -} - -static int mpam_ris_get_affinity(struct mpam_msc *msc, cpumask_t *affinity, - enum mpam_class_types type, - struct mpam_class *class, - struct mpam_component *comp) -{ - int err; - - switch (type) { - case MPAM_CLASS_CACHE: - err = get_cpumask_from_cache_id(comp->comp_id, class->level, - affinity); - if (err) - return err; - - if (cpumask_empty(affinity)) - pr_warn_once("%s no CPUs associated with cache node", - dev_name(&msc->pdev->dev)); - - break; - case MPAM_CLASS_MEMORY: - get_cpumask_from_node_id(comp->comp_id, affinity); - if (cpumask_empty(affinity)) - pr_warn_once("%s no CPUs associated with memory node", - dev_name(&msc->pdev->dev)); - break; - case MPAM_CLASS_UNKNOWN: - return 0; - } - - cpumask_and(affinity, affinity, &msc->accessibility); - - return 0; -} - -static int mpam_ris_create_locked(struct mpam_msc *msc, u8 ris_idx, - enum mpam_class_types type, u8 class_id, - int component_id, gfp_t gfp) -{ - int err; - struct mpam_msc_ris *ris; - struct mpam_class *class; - struct mpam_component *comp; - - lockdep_assert_held(&mpam_list_lock); - - if (test_and_set_bit(ris_idx, msc->ris_idxs)) - return -EBUSY; - - ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), gfp); - if (!ris) - return -ENOMEM; - - class = mpam_class_get(class_id, type, true, gfp); - if (IS_ERR(class)) - return PTR_ERR(class); - - comp = mpam_component_get(class, component_id, true, gfp); - if (IS_ERR(comp)) { - if (list_empty(&class->components)) - mpam_class_destroy(class); - return PTR_ERR(comp); - } - - err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp); - if (err) { - if (list_empty(&class->components)) - mpam_class_destroy(class); - return err; - } - - ris->ris_idx = ris_idx; - INIT_LIST_HEAD_RCU(&ris->comp_list); - INIT_LIST_HEAD_RCU(&ris->msc_list); - ris->msc = msc; - ris->comp = comp; - - cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity); - cpumask_or(&class->affinity, &class->affinity, &ris->affinity); - list_add_rcu(&ris->comp_list, &comp->ris); - list_add_rcu(&ris->msc_list, &msc->ris); - - return 0; -} - -int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, - enum mpam_class_types type, u8 class_id, int component_id) -{ - int err; - - mutex_lock(&mpam_list_lock); - err = mpam_ris_create_locked(msc, ris_idx, type, class_id, - component_id, GFP_KERNEL); - mutex_unlock(&mpam_list_lock); - - return err; -} - -static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc, - u8 ris_idx) -{ - int err; - struct mpam_msc_ris *ris, *found = ERR_PTR(-ENOENT); - - lockdep_assert_held(&mpam_list_lock); - - if (!test_bit(ris_idx, msc->ris_idxs)) { - err = mpam_ris_create_locked(msc, ris_idx, MPAM_CLASS_UNKNOWN, - 0, 0, GFP_ATOMIC); - if (err) - return ERR_PTR(err); - } - - list_for_each_entry(ris, &msc->ris, msc_list) { - if (ris->ris_idx == ris_idx) { - found = ris; - break; - } - } - - return found; -} - -static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) -{ - int err; - struct mpam_msc *msc = ris->msc; - struct mpam_props *props = &ris->props; - struct mpam_class *class = ris->comp->class; - - lockdep_assert_held(&msc->lock); - lockdep_assert_held(&msc->part_sel_lock); - - /* Cache Capacity Partitioning */ - if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) { - u32 ccap_features = mpam_read_partsel_reg(msc, CCAP_IDR); - - props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features); - if (props->cmax_wd) - mpam_set_feature(mpam_feat_ccap_part, props); - } - - /* Cache Portion partitioning */ - if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) { - u32 cpor_features = mpam_read_partsel_reg(msc, CPOR_IDR); - - props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features); - if (props->cpbm_wd) - mpam_set_feature(mpam_feat_cpor_part, props); - } - - /* Memory bandwidth partitioning */ - if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) { - u32 mbw_features = mpam_read_partsel_reg(msc, MBW_IDR); - - /* portion bitmap resolution */ - props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features); - if (props->mbw_pbm_bits && - FIELD_GET(MPAMF_MBW_IDR_HAS_PBM, mbw_features)) - mpam_set_feature(mpam_feat_mbw_part, props); - - props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features); - if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features)) - mpam_set_feature(mpam_feat_mbw_max, props); - - if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features)) - mpam_set_feature(mpam_feat_mbw_min, props); - - if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features)) - mpam_set_feature(mpam_feat_mbw_prop, props); - } - - /* Priority partitioning */ - if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) { - u32 pri_features = mpam_read_partsel_reg(msc, PRI_IDR); - - props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features); - if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) { - mpam_set_feature(mpam_feat_intpri_part, props); - if (FIELD_GET(MPAMF_PRI_IDR_INTPRI_0_IS_LOW, pri_features)) - mpam_set_feature(mpam_feat_intpri_part_0_low, props); - } - - props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features); - if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) { - mpam_set_feature(mpam_feat_dspri_part, props); - if (FIELD_GET(MPAMF_PRI_IDR_DSPRI_0_IS_LOW, pri_features)) - mpam_set_feature(mpam_feat_dspri_part_0_low, props); - } - } - - /* Performance Monitoring */ - if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) { - u32 msmon_features = mpam_read_partsel_reg(msc, MSMON_IDR); - - if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_CSU, msmon_features)) { - u32 csumonidr, discard; - - /* - * If the firmware max-nrdy-us property is missing, the - * CSU counters can't be used. Should we wait forever? - */ - err = device_property_read_u32(&msc->pdev->dev, - "arm,not-ready-us", - &discard); - - csumonidr = mpam_read_partsel_reg(msc, CSUMON_IDR); - props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr); - if (props->num_csu_mon && !err) - mpam_set_feature(mpam_feat_msmon_csu, props); - else if (props->num_csu_mon) - pr_err_once("Counters are not usable because not-ready timeout was not provided by firmware."); - } - if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) { - bool has_long; - u32 mbwumonidr = mpam_read_partsel_reg(msc, MBWUMON_IDR); - - props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumonidr); - if (props->num_mbwu_mon) - mpam_set_feature(mpam_feat_msmon_mbwu, props); - - if (FIELD_GET(MPAMF_MBWUMON_IDR_HAS_RWBW, mbwumonidr)) - mpam_set_feature(mpam_feat_msmon_mbwu_rwbw, props); - - /* - * Treat long counter and its extension, lwd as mutually - * exclusive feature bits. Though these are dependent - * fields at the implementation level, there would never - * be a need for mpam_feat_msmon_mbwu_44counter (long - * counter) and mpam_feat_msmon_mbwu_63counter (lwd) - * bits to be set together. - * - * mpam_feat_msmon_mbwu isn't treated as an exclusive - * bit as this feature bit would be used as the "front - * facing feature bit" for any checks related to mbwu - * monitors. - */ - has_long = FIELD_GET(MPAMF_MBWUMON_IDR_HAS_LONG, mbwumonidr); - if (props->num_mbwu_mon && has_long) { - if (FIELD_GET(MPAMF_MBWUMON_IDR_LWD, mbwumonidr)) - mpam_set_feature(mpam_feat_msmon_mbwu_63counter, props); - else - mpam_set_feature(mpam_feat_msmon_mbwu_44counter, props); - } - } - } - - if (FIELD_GET(MPAMF_IDR_HAS_IMPL_IDR, ris->idr)) - if (mpam_current_machine == MPAM_YITIAN710 && class->type == MPAM_CLASS_MEMORY) - mpam_set_feature(mpam_feat_impl_msmon_mbwu, props); - - /* - * RIS with PARTID narrowing don't have enough storage for one - * configuration per PARTID. If these are in a class we could use, - * reduce the supported partid_max to match the numer of intpartid. - * If the class is unknown, just ignore it. - */ - if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) && - class->type != MPAM_CLASS_UNKNOWN) { - u32 nrwidr = mpam_read_partsel_reg(msc, PARTID_NRW_IDR); - u16 partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr); - - mpam_set_feature(mpam_feat_partid_nrw, props); - msc->partid_max = min(msc->partid_max, partid_max); - } -} - -static int mpam_msc_hw_probe(struct mpam_msc *msc) -{ - u64 idr; - u16 partid_max; - u8 ris_idx, pmg_max; - struct mpam_msc_ris *ris; - - lockdep_assert_held(&msc->lock); - - spin_lock(&msc->part_sel_lock); - idr = mpam_read_partsel_reg(msc, AIDR); - if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) { - pr_err_once("%s does not match MPAM architecture v1.0\n", - dev_name(&msc->pdev->dev)); - spin_unlock(&msc->part_sel_lock); - return -EIO; - } - - idr = mpam_msc_read_idr(msc); - spin_unlock(&msc->part_sel_lock); - - msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr); - - /* Use these values so partid/pmg always starts with a valid value */ - msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); - msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr); - - for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) { - spin_lock(&msc->part_sel_lock); - __mpam_part_sel(ris_idx, 0, msc); - idr = mpam_msc_read_idr(msc); - spin_unlock(&msc->part_sel_lock); - - partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); - pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr); - msc->partid_max = min(msc->partid_max, partid_max); - msc->pmg_max = min(msc->pmg_max, pmg_max); - msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXT_ESR, idr); - - ris = mpam_get_or_create_ris(msc, ris_idx); - if (IS_ERR(ris)) { - return PTR_ERR(ris); - } - ris->idr = idr; - - spin_lock(&msc->part_sel_lock); - __mpam_part_sel(ris_idx, 0, msc); - mpam_ris_hw_probe(ris); - spin_unlock(&msc->part_sel_lock); - } - - spin_lock(&partid_max_lock); - mpam_partid_max = min(mpam_partid_max, msc->partid_max); - mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max); - spin_unlock(&partid_max_lock); - - msc->probed = true; - - return 0; -} - -struct mon_read -{ - struct mpam_msc_ris *ris; - struct mon_cfg *ctx; - enum mpam_device_features type; - u64 *val; - int err; -}; - -static bool mpam_ris_has_mbwu_long_counter(struct mpam_msc_ris *ris) -{ - return (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props) || - mpam_has_feature(mpam_feat_msmon_mbwu_44counter, &ris->props)); -} - -static u64 mpam_msc_read_mbwu_l(struct mpam_msc *msc) -{ - int retry = 3; - u32 mbwu_l_low; - u64 mbwu_l_high1, mbwu_l_high2; - - lockdep_assert_held_once(&msc->mon_sel_lock); - - WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); - WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); - - mbwu_l_high2 = readl_relaxed(msc->mapped_hwpage + MSMON_MBWU_L + 4); - do { - mbwu_l_high1 = mbwu_l_high2; - mbwu_l_low = readl_relaxed(msc->mapped_hwpage + MSMON_MBWU_L); - mbwu_l_high2 = readl_relaxed(msc->mapped_hwpage + MSMON_MBWU_L + 4); - - retry--; - } while (mbwu_l_high1 != mbwu_l_high2 && retry > 0); - - if (mbwu_l_high2 == mbwu_l_high1) - return (mbwu_l_high1 << 32) | mbwu_l_low; - return MSMON___NRDY_L; -} - -static void mpam_msc_zero_mbwu_l(struct mpam_msc *msc) -{ - lockdep_assert_held_once(&msc->mon_sel_lock); - - WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); - WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); - - writel_relaxed(0, msc->mapped_hwpage + MSMON_MBWU_L); - writel_relaxed(0, msc->mapped_hwpage + MSMON_MBWU_L + 4); -} - -static void gen_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val, - u32 *flt_val) -{ - struct mon_cfg *ctx = m->ctx; - - switch (m->type) { - case mpam_feat_msmon_csu: - *ctl_val = MSMON_CFG_MBWU_CTL_TYPE_CSU; - break; - case mpam_feat_msmon_mbwu: - *ctl_val = MSMON_CFG_MBWU_CTL_TYPE_MBWU; - break; - default: - return; - } - - /* - * For CSU counters its implementation-defined what happens when not - * filtering by partid. - */ - *ctl_val |= MSMON_CFG_x_CTL_MATCH_PARTID; - - *flt_val = FIELD_PREP(MSMON_CFG_MBWU_FLT_PARTID, ctx->partid); - *flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts); - if (m->ctx->match_pmg) { - *ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG; - *flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_PMG, ctx->pmg); - } - - if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props)) - *flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts); -} - -static void read_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val, - u32 *flt_val) -{ - struct mpam_msc *msc = m->ris->msc; - - switch (m->type) { - case mpam_feat_msmon_csu: - *ctl_val = mpam_read_monsel_reg(msc, CFG_CSU_CTL); - *flt_val = mpam_read_monsel_reg(msc, CFG_CSU_FLT); - break; - case mpam_feat_msmon_mbwu: - *ctl_val = mpam_read_monsel_reg(msc, CFG_MBWU_CTL); - *flt_val = mpam_read_monsel_reg(msc, CFG_MBWU_FLT); - break; - default: - return; - } -} - -static void write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val, - u32 flt_val) -{ - struct mpam_msc *msc = m->ris->msc; - struct msmon_mbwu_state *mbwu_state; - - /* - * Write the ctl_val with the enable bit cleared, reset the counter, - * then enable counter. - */ - switch (m->type) { - case mpam_feat_msmon_csu: - mpam_write_monsel_reg(msc, CFG_CSU_FLT, flt_val); - mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val); - mpam_write_monsel_reg(msc, CSU, 0); - mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val|MSMON_CFG_x_CTL_EN); - break; - case mpam_feat_msmon_mbwu: - mpam_write_monsel_reg(msc, CFG_MBWU_FLT, flt_val); - mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val); - - if (mpam_ris_has_mbwu_long_counter(m->ris)) - mpam_msc_zero_mbwu_l(m->ris->msc); - else - mpam_write_monsel_reg(msc, MBWU, 0); - - mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val|MSMON_CFG_x_CTL_EN); - - mbwu_state = &m->ris->mbwu_state[m->ctx->mon]; - if (mbwu_state) - mbwu_state->prev_val = 0; - - break; - default: - return; - } -} - -static u64 mpam_msmon_overflow_val(struct mpam_msc_ris *ris) -{ - /* TODO: implement scaling counters */ - if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props)) - return GENMASK_ULL(62, 0); - else if (mpam_has_feature(mpam_feat_msmon_mbwu_44counter, &ris->props)) - return GENMASK_ULL(43, 0); - else - return GENMASK_ULL(30, 0); -} - -static void __ris_msmon_read(void *arg) -{ - bool nrdy = false; - unsigned long flags; - bool config_mismatch; - struct mon_read *m = arg; - u64 now, overflow_val = 0; - struct mon_cfg *ctx = m->ctx; - bool reset_on_next_read = false; - struct mpam_msc_ris *ris = m->ris; - struct mpam_msc *msc = m->ris->msc; - struct msmon_mbwu_state *mbwu_state; - u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt; - - lockdep_assert_held(&msc->lock); - - spin_lock_irqsave(&msc->mon_sel_lock, flags); - mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) | - FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); - mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); - - if (m->type == mpam_feat_msmon_mbwu) { - mbwu_state = &ris->mbwu_state[ctx->mon]; - if (mbwu_state) { - reset_on_next_read = mbwu_state->reset_on_next_read; - mbwu_state->reset_on_next_read = false; - } - } - - /* - * Read the existing configuration to avoid re-writing the same values. - * This saves waiting for 'nrdy' on subsequent reads. - */ - read_msmon_ctl_flt_vals(m, &cur_ctl, &cur_flt); - gen_msmon_ctl_flt_vals(m, &ctl_val, &flt_val); - config_mismatch = cur_flt != flt_val || - cur_ctl != (ctl_val | MSMON_CFG_x_CTL_EN); - - if (config_mismatch || reset_on_next_read) - write_msmon_ctl_flt_vals(m, ctl_val, flt_val); - - switch (m->type) { - case mpam_feat_msmon_csu: - now = mpam_read_monsel_reg(msc, CSU); - nrdy = now & MSMON___NRDY; - now = FIELD_GET(MSMON___VALUE, now); - break; - case mpam_feat_msmon_mbwu: - /* - * If long or lwd counters are supported, use them, else revert - * to the 32 bit counter. - */ - if (mpam_ris_has_mbwu_long_counter(ris)) { - now = mpam_msc_read_mbwu_l(msc); - nrdy = now & MSMON___NRDY_L; - if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props)) - now = FIELD_GET(MSMON___LWD_VALUE, now); - else - now = FIELD_GET(MSMON___L_VALUE, now); - } else { - now = mpam_read_monsel_reg(msc, MBWU); - nrdy = now & MSMON___NRDY; - now = FIELD_GET(MSMON___VALUE, now); - } - - if (nrdy) - break; - - if (!mbwu_state) - break; - - /* Add any pre-overflow value to the mbwu_state->val */ - if (mbwu_state->prev_val > now) - overflow_val = mpam_msmon_overflow_val(ris) - mbwu_state->prev_val; - - mbwu_state->prev_val = now; - mbwu_state->correction += overflow_val; - - /* Include bandwidth consumed before the last hardware reset */ - now += mbwu_state->correction; - break; - default: - return; - } - spin_unlock_irqrestore(&msc->mon_sel_lock, flags); - - if (nrdy) { - m->err = -EBUSY; - return; - } - - *(m->val) += now; -} - -static void __ris_impl_msmon_read(void *arg) -{ - unsigned long flags; - struct mon_read *m = arg; - u64 mb_val = 0; - struct mon_cfg *ctx = m->ctx; - struct mpam_msc *msc = m->ris->msc; - u32 custom_reg_base_addr, cycle, val; - - lockdep_assert_held(&msc->lock); - if (m->type != mpam_feat_impl_msmon_mbwu) - return; - - /* Other machine can extend this function */ - if (mpam_current_machine != MPAM_YITIAN710) - return; - - spin_lock_irqsave(&msc->part_sel_lock, flags); - - __mpam_write_reg(msc, MPAMCFG_PART_SEL, ctx->mon); - - custom_reg_base_addr = __mpam_read_reg(msc, MPAMF_IMPL_IDR); - - cycle = __mpam_read_reg(msc, custom_reg_base_addr + MPAMF_CUST_WINDW_OFFSET); - val = __mpam_read_reg(msc, custom_reg_base_addr + MPAMF_CUST_MBWC_OFFSET); - - spin_unlock_irqrestore(&msc->part_sel_lock, flags); - - if (val & MSMON___NRDY) { - m->err = -EBUSY; - return; - } - - mb_val = MBWU_GET(val); - - mb_val = mb_val * 32 * ddrc_freq * 1000000 / cycle; /* B/s */ - *(m->val) += mb_val; -} - -static int _msmon_read(struct mpam_component *comp, struct mon_read *arg) -{ - int err, idx; - struct mpam_msc *msc; - struct mpam_msc_ris *ris; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(ris, &comp->ris, comp_list) { - arg->ris = ris; - - msc = ris->msc; - mutex_lock(&msc->lock); - if (arg->type == mpam_feat_msmon_csu || - arg->type == mpam_feat_msmon_mbwu) - err = smp_call_function_any(&msc->accessibility, - __ris_msmon_read, arg, true); - else if (arg->type == mpam_feat_impl_msmon_mbwu) - err = smp_call_function_any(&msc->accessibility, - __ris_impl_msmon_read, arg, true); - else - err = -EOPNOTSUPP; - mutex_unlock(&msc->lock); - if (!err && arg->err) - err = arg->err; - if (err) - break; - } - srcu_read_unlock(&mpam_srcu, idx); - - return err; -} - -int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx, - enum mpam_device_features type, u64 *val) -{ - int err; - struct mon_read arg; - u64 wait_jiffies = 0; - struct mpam_props *cprops = &comp->class->props; - - might_sleep(); - - if (!mpam_is_enabled()) - return -EIO; - - if (!mpam_has_feature(type, cprops)) - return -EOPNOTSUPP; - - memset(&arg, 0, sizeof(arg)); - arg.ctx = ctx; - arg.type = type; - arg.val = val; - *val = 0; - - err = _msmon_read(comp, &arg); - if (err == -EBUSY) - wait_jiffies = usecs_to_jiffies(comp->class->nrdy_usec); - - while (wait_jiffies) - wait_jiffies = schedule_timeout_uninterruptible(wait_jiffies); - - if (err == -EBUSY) { - memset(&arg, 0, sizeof(arg)); - arg.ctx = ctx; - arg.type = type; - arg.val = val; - *val = 0; - - err = _msmon_read(comp, &arg); - } - - return err; -} - -void mpam_msmon_reset_all_mbwu(struct mpam_component *comp) -{ - int idx, i; - unsigned long flags; - struct mpam_msc *msc; - struct mpam_msc_ris *ris; - - if (!mpam_is_enabled()) - return; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(ris, &comp->ris, comp_list) { - if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) - continue; - - msc = ris->msc; - spin_lock_irqsave(&msc->mon_sel_lock, flags); - for (i = 0; i < ris->props.num_mbwu_mon; i++) { - ris->mbwu_state[i].correction = 0; - ris->mbwu_state[i].reset_on_next_read = true; - } - spin_unlock_irqrestore(&msc->mon_sel_lock, flags); - } - srcu_read_unlock(&mpam_srcu, idx); -} - -void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx) -{ - int idx; - unsigned long flags; - struct mpam_msc *msc; - struct mpam_msc_ris *ris; - - if (!mpam_is_enabled()) - return; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(ris, &comp->ris, comp_list) { - if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) - continue; - - msc = ris->msc; - spin_lock_irqsave(&msc->mon_sel_lock, flags); - ris->mbwu_state[ctx->mon].correction = 0; - ris->mbwu_state[ctx->mon].reset_on_next_read = true; - spin_unlock_irqrestore(&msc->mon_sel_lock, flags); - } - srcu_read_unlock(&mpam_srcu, idx); -} - -static void mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd) -{ - u32 num_words, msb; - u32 bm = ~0; - int i; - - lockdep_assert_held(&msc->part_sel_lock); - - /* - * Write all ~0 to all but the last 32bit-word, which may - * have fewer bits... - */ - num_words = DIV_ROUND_UP(wd, 32); - for (i = 0; i < num_words - 1; i++, reg += sizeof(bm)) - __mpam_write_reg(msc, reg, bm); - - /* - * ....and then the last (maybe) partial 32bit word. When wd is a - * multiple of 32, msb should be 31 to write a full 32bit word. - */ - msb = (wd - 1) % 32; - bm = GENMASK(msb , 0); - if (bm) - __mpam_write_reg(msc, reg, bm); -} - -static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, - struct mpam_config *cfg) -{ - u32 pri_val = 0; - u16 cmax = MPAMCFG_CMAX_CMAX; - struct mpam_msc *msc = ris->msc; - u16 bwa_fract = MPAMCFG_MBW_MAX_MAX; - struct mpam_props *rprops = &ris->props; - u16 dspri = GENMASK(rprops->dspri_wd, 0); - u16 intpri = GENMASK(rprops->intpri_wd, 0); - u32 custom_reg_base_addr; - - spin_lock(&msc->part_sel_lock); - __mpam_part_sel(ris->ris_idx, partid, msc); - - if(mpam_has_feature(mpam_feat_partid_nrw, rprops)) - mpam_write_partsel_reg(msc, INTPARTID, - (MPAMCFG_PART_SEL_INTERNAL | partid)); - - if (mpam_has_feature(mpam_feat_cpor_part, rprops)) { - if (mpam_has_feature(mpam_feat_cpor_part, cfg)) - mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); - else - mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, - rprops->cpbm_wd); - } - - if (mpam_has_feature(mpam_feat_mbw_part, rprops)) { - if (mpam_has_feature(mpam_feat_mbw_part, cfg)) - mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); - else - mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, - rprops->mbw_pbm_bits); - } - - if (mpam_has_feature(mpam_feat_mbw_min, rprops)) - mpam_write_partsel_reg(msc, MBW_MIN, 0); - - if (mpam_has_feature(mpam_feat_mbw_max, rprops)) { - if (mpam_has_feature(mpam_feat_mbw_max, cfg)) - mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); - else - mpam_write_partsel_reg(msc, MBW_MAX, bwa_fract); - } - - if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) - mpam_write_partsel_reg(msc, MBW_PROP, bwa_fract); - - if (mpam_has_feature(mpam_feat_ccap_part, rprops)) - mpam_write_partsel_reg(msc, CMAX, cmax); - - if (mpam_has_feature(mpam_feat_intpri_part, rprops) || - mpam_has_feature(mpam_feat_dspri_part, rprops)) { - /* aces high? */ - if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops)) - intpri = 0; - if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops)) - dspri = 0; - - if (mpam_has_feature(mpam_feat_intpri_part, rprops)) - pri_val |= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri); - if (mpam_has_feature(mpam_feat_dspri_part, rprops)) - pri_val |= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri); - - mpam_write_partsel_reg(msc, PRI, pri_val); - } - - if (FIELD_GET(MPAMF_IDR_HAS_IMPL_IDR, ris->idr)) { - if (mpam_current_machine == MPAM_YITIAN710) { - custom_reg_base_addr = __mpam_read_reg(msc, MPAMF_IMPL_IDR); - __mpam_write_reg(msc, custom_reg_base_addr + - MPAMF_CUST_WINDW_OFFSET, - MBWU_WINWD_MAX); - } - } - - spin_unlock(&msc->part_sel_lock); -} - -struct reprogram_ris { - struct mpam_msc_ris *ris; - struct mpam_config *cfg; -}; - -/* Call with MSC lock held */ -static int mpam_reprogram_ris(void *_arg) -{ - u16 partid, partid_max; - struct reprogram_ris *arg = _arg; - struct mpam_msc_ris *ris = arg->ris; - struct mpam_config *cfg = arg->cfg; - - if (ris->in_reset_state) - return 0; - - spin_lock(&partid_max_lock); - partid_max = mpam_partid_max; - spin_unlock(&partid_max_lock); - for (partid = 0; partid <= partid_max; partid++) - mpam_reprogram_ris_partid(ris, partid, cfg); - - return 0; -} - -static int mpam_restore_mbwu_state(void *_ris) -{ - int i; - struct mon_read mwbu_arg; - struct mpam_msc_ris *ris = _ris; - - for (i = 0; i < ris->props.num_mbwu_mon; i++) { - if (ris->mbwu_state[i].enabled) { - mwbu_arg.ris = ris; - mwbu_arg.ctx = &ris->mbwu_state[i].cfg; - mwbu_arg.type = mpam_feat_msmon_mbwu; - - __ris_msmon_read(&mwbu_arg); - } - } - - return 0; -} - -static int mpam_save_mbwu_state(void *arg) -{ - int i; - u64 val; - struct mon_cfg *cfg; - unsigned long flags; - u32 cur_flt, cur_ctl, mon_sel; - struct mpam_msc_ris *ris = arg; - struct mpam_msc *msc = ris->msc; - struct msmon_mbwu_state *mbwu_state; - - for (i = 0; i < ris->props.num_mbwu_mon; i++) { - mbwu_state = &ris->mbwu_state[i]; - cfg = &mbwu_state->cfg; - - spin_lock_irqsave(&msc->mon_sel_lock, flags); - mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, i) | - FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); - mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); - - cur_flt = mpam_read_monsel_reg(msc, CFG_MBWU_FLT); - cur_ctl = mpam_read_monsel_reg(msc, CFG_MBWU_CTL); - mpam_write_monsel_reg(msc, CFG_MBWU_CTL, 0); - - if (mpam_ris_has_mbwu_long_counter(ris)) { - val = mpam_msc_read_mbwu_l(msc); - mpam_msc_zero_mbwu_l(msc); - } else { - val = mpam_read_monsel_reg(msc, MBWU); - mpam_write_monsel_reg(msc, MBWU, 0); - } - - cfg->mon = i; - cfg->pmg = FIELD_GET(MSMON_CFG_MBWU_FLT_PMG, cur_flt); - cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl); - cfg->partid = FIELD_GET(MSMON_CFG_MBWU_FLT_PARTID, cur_flt); - mbwu_state->correction += val; - mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl); - spin_unlock_irqrestore(&msc->mon_sel_lock, flags); - } - - return 0; -} - -/* - * Called via smp_call_on_cpu() to prevent migration, while still being - * pre-emptible. - */ -static int mpam_reset_ris(void *arg) -{ - struct mpam_msc_ris *ris = arg; - struct reprogram_ris reprogram_arg; - struct mpam_config empty_cfg = { 0 }; - - if (ris->in_reset_state) - return 0; - - reprogram_arg.ris = ris; - reprogram_arg.cfg = &empty_cfg; - - mpam_reprogram_ris(&reprogram_arg); - - return 0; -} - -/* - * Get the preferred CPU for this MSC. If it is accessible from this CPU, - * this CPU is preferred. This can be preempted/migrated, it will only result - * in more work. - */ -static int mpam_get_msc_preferred_cpu(struct mpam_msc *msc) -{ - int cpu = raw_smp_processor_id(); - - if (cpumask_test_cpu(cpu, &msc->accessibility)) - return cpu; - - return cpumask_first_and(&msc->accessibility, cpu_online_mask); -} - -static int mpam_touch_msc(struct mpam_msc *msc, int (*fn)(void *a), void *arg) -{ - lockdep_assert_irqs_enabled(); - lockdep_assert_cpus_held(); - lockdep_assert_held(&msc->lock); - - return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true); -} - -static void mpam_reset_msc(struct mpam_msc *msc, bool online) -{ - int idx; - struct mpam_msc_ris *ris; - - lockdep_assert_held(&msc->lock); - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(ris, &msc->ris, msc_list) { - mpam_touch_msc(msc, &mpam_reset_ris, ris); - - /* - * Set in_reset_state when coming online. The reset state - * for non-zero partid may be lost while the CPUs are offline. - */ - ris->in_reset_state = online; - - if (mpam_is_enabled() && !online) - mpam_touch_msc(msc, &mpam_save_mbwu_state, ris); - } - srcu_read_unlock(&mpam_srcu, idx); -} - -static void mpam_reprogram_msc(struct mpam_msc *msc) -{ - int idx; - u16 partid; - bool reset; - struct mpam_config *cfg; - struct mpam_msc_ris *ris; - - lockdep_assert_held(&msc->lock); - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(ris, &msc->ris, msc_list) { - if (!mpam_is_enabled() && !ris->in_reset_state) { - mpam_touch_msc(msc, &mpam_reset_ris, ris); - ris->in_reset_state = true; - continue; - } - - reset = true; - for (partid = 0; partid <= mpam_partid_max; partid++) { - cfg = &ris->comp->cfg[partid]; - if (cfg->features) - reset = false; - - mpam_reprogram_ris_partid(ris, partid, cfg); - } - ris->in_reset_state = reset; - - if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) - mpam_touch_msc(msc, &mpam_restore_mbwu_state, ris); - } - srcu_read_unlock(&mpam_srcu, idx); -} - -static void _enable_percpu_irq(void *_irq) -{ - int *irq = _irq; - enable_percpu_irq(*irq, IRQ_TYPE_NONE); -} - -static int mpam_cpu_online(unsigned int cpu) -{ - int idx; - struct mpam_msc *msc; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(msc, &mpam_all_msc, glbl_list) { - if (!cpumask_test_cpu(cpu, &msc->accessibility)) - continue; - - mutex_lock(&msc->lock); - if (msc->reenable_error_ppi) - _enable_percpu_irq(&msc->reenable_error_ppi); - - if (atomic_fetch_inc(&msc->online_refs) == 0) - mpam_reprogram_msc(msc); - mutex_unlock(&msc->lock); - } - srcu_read_unlock(&mpam_srcu, idx); - - if (mpam_is_enabled()) - mpam_resctrl_online_cpu(cpu); - - return 0; -} - -/* Before mpam is enabled, try to probe new MSC */ -static int mpam_discovery_cpu_online(unsigned int cpu) -{ - int err = 0; - struct mpam_msc *msc; - bool new_device_probed = false; - - if (mpam_is_enabled()) - return 0; - - mutex_lock(&mpam_list_lock); - list_for_each_entry(msc, &mpam_all_msc, glbl_list) { - if (!cpumask_test_cpu(cpu, &msc->accessibility)) - continue; - - mutex_lock(&msc->lock); - if (!msc->probed) - err = mpam_msc_hw_probe(msc); - mutex_unlock(&msc->lock); - - if (!err) - new_device_probed = true; - else - break; // mpam_broken - } - mutex_unlock(&mpam_list_lock); - - if (new_device_probed && !err) - schedule_work(&mpam_enable_work); - - if (err < 0) - return err; - - return mpam_cpu_online(cpu); -} - -static int mpam_cpu_offline(unsigned int cpu) -{ - int idx; - struct mpam_msc *msc; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(msc, &mpam_all_msc, glbl_list) { - if (!cpumask_test_cpu(cpu, &msc->accessibility)) - continue; - - mutex_lock(&msc->lock); - if (msc->reenable_error_ppi) - disable_percpu_irq(msc->reenable_error_ppi); - - if (atomic_dec_and_test(&msc->online_refs)) - mpam_reset_msc(msc, false); - mutex_unlock(&msc->lock); - } - srcu_read_unlock(&mpam_srcu, idx); - - if (mpam_is_enabled()) - mpam_resctrl_offline_cpu(cpu); - - return 0; -} - -static void mpam_register_cpuhp_callbacks(int (*online)(unsigned int online)) -{ - mutex_lock(&mpam_cpuhp_state_lock); - mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mpam:online", - online, mpam_cpu_offline); - if (mpam_cpuhp_state <= 0) { - pr_err("Failed to register cpuhp callbacks"); - mpam_cpuhp_state = 0; - } - mutex_unlock(&mpam_cpuhp_state_lock); -} - -static int __setup_ppi(struct mpam_msc *msc) -{ - int cpu; - - msc->error_dev_id = alloc_percpu_gfp(struct mpam_msc *, GFP_KERNEL); - if (!msc->error_dev_id) - return -ENOMEM; - - for_each_cpu(cpu, &msc->accessibility) { - struct mpam_msc *empty = *per_cpu_ptr(msc->error_dev_id, cpu); - if (empty != NULL) { - pr_err_once("%s shares PPI with %s!\n", dev_name(&msc->pdev->dev), - dev_name(&empty->pdev->dev)); - return -EBUSY; - } - *per_cpu_ptr(msc->error_dev_id, cpu) = msc; - } - - return 0; -} - -static int mpam_msc_setup_error_irq(struct mpam_msc *msc) -{ - int irq; - - irq = platform_get_irq_byname_optional(msc->pdev, "error"); - if (irq <= 0) - return 0; - - /* Allocate and initialise the percpu device pointer for PPI */ - if (irq_is_percpu(irq)) - - return __setup_ppi(msc); - - /* sanity check: shared interrupts can be routed anywhere? */ - if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) { - pr_err_once("msc:%u is a private resource with a shared error interrupt", - msc->id); - return -EINVAL; - } - - return 0; -} - -static enum mpam_machine_type mpam_dt_get_machine_type(void) -{ - /* FIXME: not supported yet */ - return MPAM_DEFAULT_MACHINE; -} - -static int mpam_dt_count_msc(void) -{ - int count = 0; - struct device_node *np; - - for_each_compatible_node(np, NULL, "arm,mpam-msc") - count++; - - return count; -} - -static int mpam_dt_parse_resource(struct mpam_msc *msc, struct device_node *np, - u32 ris_idx) -{ - int err = 0; - u32 level = 0; - unsigned long cache_id; - struct device_node *cache; - - do { - if (of_device_is_compatible(np, "arm,mpam-cache")) { - cache = of_parse_phandle(np, "arm,mpam-device", 0); - if (!cache) { - pr_err("Failed to read phandle\n"); - break; - } - } else if (of_device_is_compatible(np->parent, "cache")) { - cache = np->parent; - } else { - /* For now, only caches are supported */ - cache = NULL; - break; - } - - err = of_property_read_u32(cache, "cache-level", &level); - if (err) { - pr_err("Failed to read cache-level\n"); - break; - } - - cache_id = cache_of_get_id(cache); - if (cache_id == ~0UL) { - err = -ENOENT; - break; - } - - err = mpam_ris_create(msc, ris_idx, MPAM_CLASS_CACHE, level, - cache_id); - } while (0); - of_node_put(cache); - - return err; -} - - -static int mpam_dt_parse_resources(struct mpam_msc *msc, void *ignored) -{ - int err, num_ris = 0; - const u32 *ris_idx_p; - struct device_node *iter, *np; - - np = msc->pdev->dev.of_node; - for_each_child_of_node(np, iter) { - ris_idx_p = of_get_property(iter, "reg", NULL); - if (ris_idx_p) { - num_ris++; - err = mpam_dt_parse_resource(msc, iter, *ris_idx_p); - if (err) { - of_node_put(iter); - return err; - } - } - } - - if (!num_ris) - mpam_dt_parse_resource(msc, np, 0); - - return err; -} - -static int get_msc_affinity(struct mpam_msc *msc) -{ - struct device_node *parent; - u32 affinity_id; - int err; - - if (!acpi_disabled) { - err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity", - &affinity_id); - if (err) { - cpumask_copy(&msc->accessibility, cpu_possible_mask); - err = 0; - } else { - err = acpi_pptt_get_cpus_from_container(affinity_id, - &msc->accessibility); - } - - return err; - } - - /* This depends on the path to of_node */ - parent = of_get_parent(msc->pdev->dev.of_node); - if (parent == of_root) { - cpumask_copy(&msc->accessibility, cpu_possible_mask); - err = 0; - } else { - if (of_device_is_compatible(parent, "cache")) { - err = get_cpumask_from_cache(parent, - &msc->accessibility); - } else { - err = -EINVAL; - pr_err("Cannot determine accessibility of MSC: %s\n", - dev_name(&msc->pdev->dev)); - } - } - of_node_put(parent); - - return err; -} - -static int fw_num_msc; - -static void mpam_pcc_rx_callback(struct mbox_client *cl, void *msg) -{ - /* TODO: wake up tasks blocked on this MSC's PCC channel */ -} - -static int mpam_msc_drv_remove(struct platform_device *pdev) -{ - struct mpam_msc *msc = platform_get_drvdata(pdev); - - if (!msc) - return 0; - - mutex_lock(&mpam_list_lock); - mpam_num_msc--; - platform_set_drvdata(pdev, NULL); - list_del_rcu(&msc->glbl_list); - mpam_msc_destroy(msc); - synchronize_srcu(&mpam_srcu); - mutex_unlock(&mpam_list_lock); - - return 0; -} - -static int mpam_msc_drv_probe(struct platform_device *pdev) -{ - int err; - pgprot_t prot; - void * __iomem io; - struct mpam_msc *msc; - struct resource *msc_res; - void *plat_data = pdev->dev.platform_data; - - mutex_lock(&mpam_list_lock); - do { - msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL); - if (!msc) { - err = -ENOMEM; - break; - } - - INIT_LIST_HEAD_RCU(&msc->glbl_list); - msc->pdev = pdev; - - err = device_property_read_u32(&pdev->dev, "arm,not-ready-us", - &msc->nrdy_usec); - if (err) { - /* This will prevent CSU monitors being usable */ - msc->nrdy_usec = 0; - } - - err = get_msc_affinity(msc); - if (err) - break; - if (cpumask_empty(&msc->accessibility)) { - pr_err_once("msc:%u is not accessible from any CPU!", - msc->id); - err = -EINVAL; - break; - } - - mutex_init(&msc->lock); - msc->id = mpam_num_msc++; - INIT_LIST_HEAD_RCU(&msc->ris); - spin_lock_init(&msc->part_sel_lock); - spin_lock_init(&msc->mon_sel_lock); - - err = mpam_msc_setup_error_irq(msc); - if (err) { - msc = ERR_PTR(err); - break; - } - - if (device_property_read_u32(&pdev->dev, "pcc-channel", - &msc->pcc_subspace_id)) - msc->iface = MPAM_IFACE_MMIO; - else - msc->iface = MPAM_IFACE_PCC; - - if (msc->iface == MPAM_IFACE_MMIO) { - io = devm_platform_get_and_ioremap_resource(pdev, 0, - &msc_res); - if (IS_ERR(io)) { - pr_err("Failed to map MSC base address\n"); - err = PTR_ERR(io); - break; - } - msc->mapped_hwpage_sz = msc_res->end - msc_res->start + 1; - msc->mapped_hwpage = io; - if (msc->mapped_hwpage_sz < MPAM_MIN_MMIO_SIZE) { - pr_err("MSC MMIO space size is too small\n"); - err = -EINVAL; - break; - } - } else if (msc->iface == MPAM_IFACE_PCC) { - msc->pcc_cl.dev = &pdev->dev; - msc->pcc_cl.rx_callback = mpam_pcc_rx_callback; - msc->pcc_cl.tx_block = false; - msc->pcc_cl.tx_tout = 1000; /* 1s */ - msc->pcc_cl.knows_txdone = false; - - msc->pcc_chan = pcc_mbox_request_channel(&msc->pcc_cl, - msc->pcc_subspace_id); - if (IS_ERR(msc->pcc_chan)) { - pr_err("Failed to request MSC PCC channel\n"); - err = PTR_ERR(msc->pcc_chan); - break; - } - - prot = __acpi_get_mem_attribute(msc->pcc_chan->shmem_base_addr); - io = ioremap_prot(msc->pcc_chan->shmem_base_addr, - msc->pcc_chan->shmem_size, pgprot_val(prot)); - if (IS_ERR(io)) { - pr_err("Failed to map MSC base address\n"); - pcc_mbox_free_channel(msc->pcc_chan); - err = PTR_ERR(io); - break; - } - - /* TODO: issue a read to update the registers */ - - msc->mapped_hwpage_sz = msc->pcc_chan->shmem_size; - msc->mapped_hwpage = io + sizeof(struct acpi_pcct_shared_memory); - } - - list_add_rcu(&msc->glbl_list, &mpam_all_msc); - platform_set_drvdata(pdev, msc); - } while (0); - mutex_unlock(&mpam_list_lock); - - if (!err) { - /* Create RIS entries described by firmware */ - if (!acpi_disabled) - err = acpi_mpam_parse_resources(msc, plat_data); - else - err = mpam_dt_parse_resources(msc, plat_data); - } - - if (err) - mpam_msc_drv_remove(pdev); - - if (!err && fw_num_msc == mpam_num_msc) - mpam_register_cpuhp_callbacks(&mpam_discovery_cpu_online); - - return err; -} - -/* - * If a resource doesn't match class feature/configuration, do the right thing. - * For 'num' properties we can just take the minimum. - * For properties where the mismatched unused bits would make a difference, we - * nobble the class feature, as we can't configure all the resources. - * e.g. The L3 cache is composed of two resources with 13 and 17 portion - * bitmaps respectively. - */ -static void -__resource_props_mismatch(struct mpam_msc_ris *ris, struct mpam_class *class) -{ - struct mpam_props *cprops = &class->props; - struct mpam_props *rprops = &ris->props; - - lockdep_assert_held(&mpam_list_lock); /* we modify class */ - - /* Clear missing features */ - cprops->features &= rprops->features; - - /* Clear incompatible features */ - if (cprops->cpbm_wd != rprops->cpbm_wd) - mpam_clear_feature(mpam_feat_cpor_part, &cprops->features); - if (cprops->mbw_pbm_bits != rprops->mbw_pbm_bits) - mpam_clear_feature(mpam_feat_mbw_part, &cprops->features); - - /* bwa_wd is a count of bits, fewer bits means less precision */ - if (cprops->bwa_wd != rprops->bwa_wd) - cprops->bwa_wd = min(cprops->bwa_wd, rprops->bwa_wd); - - /* For num properties, take the minimum */ - if (cprops->num_csu_mon != rprops->num_csu_mon) - cprops->num_csu_mon = min(cprops->num_csu_mon, rprops->num_csu_mon); - if (cprops->num_mbwu_mon != rprops->num_mbwu_mon) - cprops->num_mbwu_mon = min(cprops->num_mbwu_mon, rprops->num_mbwu_mon); - - if (cprops->intpri_wd != rprops->intpri_wd) - cprops->intpri_wd = min(cprops->intpri_wd, rprops->intpri_wd); - if (cprops->dspri_wd != rprops->dspri_wd) - cprops->dspri_wd = min(cprops->dspri_wd, rprops->dspri_wd); - - /* {int,ds}pri may not have differing 0-low behaviour */ - if (mpam_has_feature(mpam_feat_intpri_part_0_low, cprops) != - mpam_has_feature(mpam_feat_intpri_part_0_low, rprops)) - mpam_clear_feature(mpam_feat_intpri_part, &cprops->features); - if (mpam_has_feature(mpam_feat_dspri_part_0_low, cprops) != - mpam_has_feature(mpam_feat_dspri_part_0_low, rprops)) - mpam_clear_feature(mpam_feat_dspri_part, &cprops->features); -} - -/* - * Copy the first component's first resources's properties and features to the - * class. __resource_props_mismatch() will remove conflicts. - * It is not possible to have a class with no components, or a component with - * no resources. - */ -static void mpam_enable_init_class_features(struct mpam_class *class) -{ - struct mpam_msc_ris *ris; - struct mpam_component *comp; - - comp = list_first_entry_or_null(&class->components, - struct mpam_component, class_list); - if (WARN_ON(!comp)) - return; - - ris = list_first_entry_or_null(&comp->ris, - struct mpam_msc_ris, comp_list); - if (WARN_ON(!ris)) - return; - - class->props = ris->props; -} - -/* Merge all the common resource features into class. */ -static void mpam_enable_merge_features(void) -{ - struct mpam_msc_ris *ris; - struct mpam_class *class; - struct mpam_component *comp; - - lockdep_assert_held(&mpam_list_lock); - - list_for_each_entry(class, &mpam_classes, classes_list) { - mpam_enable_init_class_features(class); - - list_for_each_entry(comp, &class->components, class_list) { - list_for_each_entry(ris, &comp->ris, comp_list) { - __resource_props_mismatch(ris, class); - - class->nrdy_usec = max(class->nrdy_usec, - ris->msc->nrdy_usec); - } - } - } -} - -static char *mpam_errcode_names[16] = { - [0] = "No error", - [1] = "PARTID_SEL_Range", - [2] = "Req_PARTID_Range", - [3] = "MSMONCFG_ID_RANGE", - [4] = "Req_PMG_Range", - [5] = "Monitor_Range", - [6] = "intPARTID_Range", - [7] = "Unexpected_INTERNAL", - [8] = "Undefined_RIS_PART_SEL", - [9] = "RIS_No_Control", - [10] = "Undefined_RIS_MON_SEL", - [11] = "RIS_No_Monitor", - [12 ... 15] = "Reserved" -}; - -static int mpam_enable_msc_ecr(void *_msc) -{ - struct mpam_msc *msc = _msc; - - writel_relaxed(1, msc->mapped_hwpage + MPAMF_ECR); - - return 0; -} - -static int mpam_disable_msc_ecr(void *_msc) -{ - struct mpam_msc *msc = _msc; - - writel_relaxed(0, msc->mapped_hwpage + MPAMF_ECR); - - return 0; -} - -static irqreturn_t __mpam_irq_handler(int irq, struct mpam_msc *msc) -{ - u64 reg; - u16 partid; - u8 errcode, pmg, ris; - - if (WARN_ON_ONCE(!msc) || - WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), - &msc->accessibility))) - return IRQ_NONE; - - reg = mpam_msc_read_esr(msc); - - errcode = FIELD_GET(MPAMF_ESR_ERRCODE, reg); - if (!errcode) - return IRQ_NONE; - - /* Clear level triggered irq */ - mpam_msc_zero_esr(msc); - - partid = FIELD_GET(MPAMF_ESR_PARTID_OR_MON, reg); - pmg = FIELD_GET(MPAMF_ESR_PMG, reg); - ris = FIELD_GET(MPAMF_ESR_RIS, reg); - - pr_err("error irq from msc:%u '%s', partid:%u, pmg: %u, ris: %u\n", - msc->id, mpam_errcode_names[errcode], partid, pmg, ris); - - /* - * To prevent this interrupt from repeatedly cancelling the scheduled - * work to disable mpam, disable the error interrupt. - */ - mpam_disable_msc_ecr(msc); - - schedule_work(&mpam_broken_work); - - return IRQ_HANDLED; -} - -static irqreturn_t mpam_ppi_handler(int irq, void *dev_id) -{ - struct mpam_msc *msc = *(struct mpam_msc **)dev_id; - - return __mpam_irq_handler(irq, msc); -} - -static irqreturn_t mpam_spi_handler(int irq, void *dev_id) -{ - struct mpam_msc *msc = dev_id; - - return __mpam_irq_handler(irq, msc); -} - -static int mpam_register_irqs(void) -{ - int err, irq; - struct mpam_msc *msc; - - lockdep_assert_cpus_held(); - lockdep_assert_held(&mpam_list_lock); - - list_for_each_entry(msc, &mpam_all_msc, glbl_list) { - irq = platform_get_irq_byname_optional(msc->pdev, "error"); - if (irq <= 0) - continue; - - /* The MPAM spec says the interrupt can be SPI, PPI or LPI */ - /* We anticipate sharing the interrupt with other MSCs */ - if (irq_is_percpu(irq)) { - err = request_percpu_irq(irq, &mpam_ppi_handler, - "mpam:msc:error", - msc->error_dev_id); - if (err) - return err; - - mutex_lock(&msc->lock); - msc->reenable_error_ppi = irq; - smp_call_function_many(&msc->accessibility, - &_enable_percpu_irq, &irq, - true); - mutex_unlock(&msc->lock); - } else { - err = devm_request_irq(&msc->pdev->dev, irq, - &mpam_spi_handler, IRQF_SHARED, - "mpam:msc:error", msc); - if (err) - return err; - } - - mutex_lock(&msc->lock); - msc->error_irq_requested = true; - mpam_touch_msc(msc, mpam_enable_msc_ecr, msc); - msc->error_irq_hw_enabled = true; - mutex_unlock(&msc->lock); - } - - return 0; -} - -static void mpam_unregister_irqs(void) -{ - int irq; - struct mpam_msc *msc; - - cpus_read_lock(); - /* take the lock as free_irq() can sleep */ - mutex_lock(&mpam_list_lock); - list_for_each_entry(msc, &mpam_all_msc, glbl_list) { - irq = platform_get_irq_byname_optional(msc->pdev, "error"); - if (irq <= 0) - continue; - - mutex_lock(&msc->lock); - if (msc->error_irq_hw_enabled) { - mpam_touch_msc(msc, mpam_disable_msc_ecr, msc); - msc->error_irq_hw_enabled = false; - } - - if (msc->error_irq_requested) { - if (irq_is_percpu(irq)) { - msc->reenable_error_ppi = 0; - free_percpu_irq(irq, msc->error_dev_id); - } else { - devm_free_irq(&msc->pdev->dev, irq, msc); - } - msc->error_irq_requested = false; - } - mutex_unlock(&msc->lock); - } - mutex_unlock(&mpam_list_lock); - cpus_read_unlock(); -} - -static void __destroy_component_cfg(struct mpam_component *comp) -{ - unsigned long flags; - struct mpam_msc_ris *ris; - struct msmon_mbwu_state *mbwu_state; - - kfree(comp->cfg); - list_for_each_entry(ris, &comp->ris, comp_list) { - mutex_lock(&ris->msc->lock); - spin_lock_irqsave(&ris->msc->mon_sel_lock, flags); - mbwu_state = ris->mbwu_state; - ris->mbwu_state = NULL; - spin_unlock_irqrestore(&ris->msc->mon_sel_lock, flags); - mutex_unlock(&ris->msc->lock); - - kfree(mbwu_state); - } -} - -static int __allocate_component_cfg(struct mpam_component *comp) -{ - unsigned long flags; - struct mpam_msc_ris *ris; - struct msmon_mbwu_state *mbwu_state; - - if (comp->cfg) - return 0; - - comp->cfg = kcalloc(mpam_partid_max + 1, sizeof(*comp->cfg), GFP_KERNEL); - if (!comp->cfg) - return -ENOMEM; - - list_for_each_entry(ris, &comp->ris, comp_list) { - if (!ris->props.num_mbwu_mon) - continue; - - mbwu_state = kcalloc(ris->props.num_mbwu_mon, - sizeof(*ris->mbwu_state), GFP_KERNEL); - if (!mbwu_state) { - __destroy_component_cfg(comp); - return -ENOMEM; - } - - mutex_lock(&ris->msc->lock); - spin_lock_irqsave(&ris->msc->mon_sel_lock, flags); - ris->mbwu_state = mbwu_state; - spin_unlock_irqrestore(&ris->msc->mon_sel_lock, flags); - mutex_unlock(&ris->msc->lock); - } - - return 0; -} - -static int mpam_allocate_config(void) -{ - int err = 0; - struct mpam_class *class; - struct mpam_component *comp; - - lockdep_assert_held(&mpam_list_lock); - - list_for_each_entry(class, &mpam_classes, classes_list) { - list_for_each_entry(comp, &class->components, class_list) { - err = __allocate_component_cfg(comp); - if (err) - return err; - } - } - - return 0; -} - -static void mpam_enable_once(void) -{ - int err; - - /* - * If all the MSC have been probed, enabling the IRQs happens next. - * That involves cross-calling to a CPU that can reach the MSC, and - * the locks must be taken in this order: - */ - cpus_read_lock(); - mutex_lock(&mpam_list_lock); - do { - mpam_enable_merge_features(); - - err = mpam_allocate_config(); - if (err) { - pr_err("Failed to allocate configuration arrays.\n"); - break; - } - - err = mpam_register_irqs(); - if (err) { - pr_warn("Failed to register irqs: %d\n", err); - break; - } - } while (0); - mutex_unlock(&mpam_list_lock); - cpus_read_unlock(); - - if (!err) { - err = mpam_resctrl_setup(); - if (err) - pr_err("Failed to initialise resctrl: %d\n", err); - } - - if (err) { - schedule_work(&mpam_broken_work); - return; - } - - mutex_lock(&mpam_cpuhp_state_lock); - cpuhp_remove_state(mpam_cpuhp_state); - mpam_cpuhp_state = 0; - mutex_unlock(&mpam_cpuhp_state_lock); - - /* - * Once the cpuhp callbacks have been changed, mpam_partid_max can no - * longer change. - */ - spin_lock(&partid_max_lock); - partid_max_published = true; - spin_unlock(&partid_max_lock); - - static_branch_enable(&mpam_enabled); - mpam_register_cpuhp_callbacks(mpam_cpu_online); - - pr_info("MPAM enabled with %u partid and %u pmg\n", - READ_ONCE(mpam_partid_max) + 1, READ_ONCE(mpam_pmg_max) + 1); -} - -void mpam_reset_class(struct mpam_class *class) -{ - int idx; - struct mpam_msc_ris *ris; - struct mpam_component *comp; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(comp, &class->components, class_list) { - memset(comp->cfg, 0, ((mpam_partid_max + 1) * sizeof(*comp->cfg))); - - list_for_each_entry_rcu(ris, &comp->ris, comp_list) { - mutex_lock(&ris->msc->lock); - mpam_touch_msc(ris->msc, mpam_reset_ris, ris); - mutex_unlock(&ris->msc->lock); - ris->in_reset_state = true; - } - } - srcu_read_unlock(&mpam_srcu, idx); -} - -/* - * Called in response to an error IRQ. - * All of MPAMs errors indicate a software bug, restore any modified - * controls to their reset values. - */ -void mpam_disable(struct work_struct *ignored) -{ - int idx; - struct mpam_class *class; - - mutex_lock(&mpam_cpuhp_state_lock); - if (mpam_cpuhp_state) { - cpuhp_remove_state(mpam_cpuhp_state); - mpam_cpuhp_state = 0; - } - mutex_unlock(&mpam_cpuhp_state_lock); - - mpam_resctrl_exit(); - - static_branch_disable(&mpam_enabled); - - mpam_unregister_irqs(); - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(class, &mpam_classes, classes_list) - mpam_reset_class(class); - srcu_read_unlock(&mpam_srcu, idx); -} - -/* - * Enable mpam once all devices have been probed. - * Scheduled by mpam_discovery_cpu_online() once all devices have been created. - * Also scheduled when new devices are probed when new CPUs come online. - */ -void mpam_enable(struct work_struct *work) -{ - static atomic_t once; - struct mpam_msc *msc; - bool all_devices_probed = true; - - mutex_lock(&mpam_list_lock); - list_for_each_entry(msc, &mpam_all_msc, glbl_list) { - mutex_lock(&msc->lock); - if (!msc->probed) - all_devices_probed = false; - mutex_unlock(&msc->lock); - - if (!all_devices_probed) - break; - } - mutex_unlock(&mpam_list_lock); - - if (all_devices_probed && !atomic_fetch_inc(&once)) - mpam_enable_once(); -} - -struct mpam_write_config_arg { - struct mpam_msc_ris *ris; - struct mpam_component *comp; - u16 partid; -}; - -static int __write_config(void *arg) -{ - struct mpam_write_config_arg *c = arg; - - mpam_reprogram_ris_partid(c->ris, c->partid, &c->comp->cfg[c->partid]); - - return 0; -} - -/* TODO: split into write_config/sync_config */ -/* TODO: add config_dirty bitmap to drive sync_config */ -int mpam_apply_config(struct mpam_component *comp, u16 partid, - struct mpam_config *cfg) -{ - struct mpam_write_config_arg arg; - struct mpam_msc_ris *ris; - int idx; - - lockdep_assert_cpus_held(); - - if (!memcmp(&comp->cfg[partid], cfg, sizeof(*cfg))) - return 0; - - comp->cfg[partid] = *cfg; - arg.comp = comp; - arg.partid = partid; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(ris, &comp->ris, comp_list) { - arg.ris = ris; - mutex_lock(&ris->msc->lock); - mpam_touch_msc(ris->msc, __write_config, &arg); - mutex_unlock(&ris->msc->lock); - } - srcu_read_unlock(&mpam_srcu, idx); - - return 0; -} - -static const struct of_device_id mpam_of_match[] = { - { .compatible = "arm,mpam-msc", }, - {}, -}; -MODULE_DEVICE_TABLE(of, mpam_of_match); - -static struct platform_driver mpam_msc_driver = { - .driver = { - .name = "mpam_msc", - .of_match_table = of_match_ptr(mpam_of_match), - }, - .probe = mpam_msc_drv_probe, - .remove = mpam_msc_drv_remove, -}; - -/* - * MSC that are hidden under caches are not created as platform devices - * as there is no cache driver. Caches are also special-cased in - * get_msc_affinity(). - */ -static void mpam_dt_create_foundling_msc(void) -{ - int err; - struct device_node *cache; - - for_each_compatible_node(cache, NULL, "cache") { - err = of_platform_populate(cache, mpam_of_match, NULL, NULL); - if (err) { - pr_err("Failed to create MSC devices under caches\n"); - } - } -} - -static int __init arm64_mpam_register_cpus(void) -{ - u64 mpamidr = read_sysreg_s(SYS_MPAMIDR_EL1); - u16 partid_max = FIELD_GET(MPAMIDR_PARTID_MAX, mpamidr); - u8 pmg_max = FIELD_GET(MPAMIDR_PMG_MAX, mpamidr); - - return mpam_register_requestor(partid_max, pmg_max); -} - -static int mpam_on; - -static int __init mpam_on_setup(char *str) -{ - mpam_on = 1; - pr_info("MPAM is supported now"); - return 1; -} -__setup("mpam_on", mpam_on_setup); - -static int __init mpam_msc_driver_init(void) -{ - if (!mpam_on) - return 0; - - bool mpam_not_available = false; - int err; - - if (!mpam_cpus_have_feature()) - return -EOPNOTSUPP; - - init_srcu_struct(&mpam_srcu); - - if (!acpi_disabled) - mpam_current_machine = acpi_mpam_get_machine_type(); - else - mpam_current_machine = mpam_dt_get_machine_type(); - - if (!acpi_disabled) - fw_num_msc = acpi_mpam_count_msc(); - else - fw_num_msc = mpam_dt_count_msc(); - - if (fw_num_msc <= 0) { - pr_err("No MSC devices found in firmware\n"); - return -EINVAL; - } - - /* - * Access MPAM system registers after MPAM ACPI table is parsed, since - * some BIOSs disable MPAM system registers accessing but export MPAM in - * ID_AA64PFR0_EL1. So we can only rely on the MPAM ACPI table to - * determine whether MPAM feature is enabled. - */ - err = arm64_mpam_register_cpus(); - if (err) - return err; - - /* - * If the MPAM CPU interface is not implemented, or reserved by - * firmware, there is no point touching the rest of the hardware. - */ - spin_lock(&partid_max_lock); - if (!partid_max_init || (!mpam_partid_max && !mpam_pmg_max)) - mpam_not_available = true; - spin_unlock(&partid_max_lock); - - if (mpam_not_available) - return 0; - - if (acpi_disabled) - mpam_dt_create_foundling_msc(); - - return platform_driver_register(&mpam_msc_driver); -} -/* Must occur after arm64_mpam_register_cpus() from arch_initcall() */ -subsys_initcall(mpam_msc_driver_init); diff --git a/drivers/platform/mpam/mpam_internal.h b/drivers/platform/mpam/mpam_internal.h deleted file mode 100644 index c46af5caa90c84b0f90a14efa1ae9d9e1f76c04a..0000000000000000000000000000000000000000 --- a/drivers/platform/mpam/mpam_internal.h +++ /dev/null @@ -1,587 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2021 Arm Ltd. - -#ifndef MPAM_INTERNAL_H -#define MPAM_INTERNAL_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -DECLARE_STATIC_KEY_FALSE(mpam_enabled); - -/* Value to indicate the allocated monitor is derived from the RMID index. */ -#define USE_RMID_IDX (U16_MAX + 1) - -/* - * Only these event configuration bits are supported. MPAM can't know if - * data is being written back, these will show up as a write. - */ -#define READS_TO_LOCAL_MEM BIT(0) -#define NON_TEMP_WRITE_TO_LOCAL_MEM BIT(2) -#define MPAM_RESTRL_EVT_CONFIG_VALID (READS_TO_LOCAL_MEM | NON_TEMP_WRITE_TO_LOCAL_MEM) - -static inline bool mpam_is_enabled(void) -{ - return static_branch_likely(&mpam_enabled); -} - -struct mpam_msc -{ - /* member of mpam_all_msc */ - struct list_head glbl_list; - - int id; - struct platform_device *pdev; - - /* Not modified after mpam_is_enabled() becomes true */ - enum mpam_msc_iface iface; - u32 pcc_subspace_id; - struct mbox_client pcc_cl; - struct pcc_mbox_chan *pcc_chan; - u32 nrdy_usec; - cpumask_t accessibility; - bool has_extd_esr; - - int reenable_error_ppi; - struct mpam_msc * __percpu *error_dev_id; - - atomic_t online_refs; - - struct mutex lock; - bool probed; - bool error_irq_requested; - bool error_irq_hw_enabled; - u16 partid_max; - u8 pmg_max; - unsigned long ris_idxs[128 / BITS_PER_LONG]; - u32 ris_max; - - /* mpam_msc_ris of this component */ - struct list_head ris; - - /* - * part_sel_lock protects access to the MSC hardware registers that are - * affected by MPAMCFG_PART_SEL. (including the ID registers) - * If needed, take msc->lock first. - */ - spinlock_t part_sel_lock; - spinlock_t mon_sel_lock; - void __iomem * mapped_hwpage; - size_t mapped_hwpage_sz; -}; - -/* - * When we compact the supported features, we don't care what they are. - * Storing them as a bitmap makes life easy. - */ -typedef u32 mpam_features_t; - -/* Bits for mpam_features_t */ -enum mpam_device_features { - mpam_feat_ccap_part = 0, - mpam_feat_cpor_part, - mpam_feat_mbw_part, - mpam_feat_mbw_min, - mpam_feat_mbw_max, - mpam_feat_mbw_prop, - mpam_feat_intpri_part, - mpam_feat_intpri_part_0_low, - mpam_feat_dspri_part, - mpam_feat_dspri_part_0_low, - mpam_feat_msmon, - mpam_feat_msmon_csu, - mpam_feat_msmon_csu_capture, - /* - * Having mpam_feat_msmon_mbwu set doesn't mean the regular 31 bit MBWU - * counter would be used. The exact counter used is decided based on the - * status of mpam_feat_msmon_mbwu_l/mpam_feat_msmon_mbwu_lwd as well. - */ - mpam_feat_msmon_mbwu, - mpam_feat_msmon_mbwu_44counter, - mpam_feat_msmon_mbwu_63counter, - mpam_feat_msmon_mbwu_capture, - mpam_feat_msmon_mbwu_rwbw, - mpam_feat_msmon_capt, - mpam_feat_impl_msmon_mbwu, - mpam_feat_partid_nrw, - MPAM_FEATURE_LAST, -}; -#define MPAM_ALL_FEATURES ((1<features) -#define mpam_set_feature(_feat, x) ((x)->features |= (1<<_feat)) - -static inline void mpam_clear_feature(enum mpam_device_features feat, - mpam_features_t *supported) -{ - *supported &= ~(1<lock. - * Changes to reset_on_next_read, prev_val and correction are protected by the - * msc's mon_sel_lock. - */ -struct msmon_mbwu_state { - bool enabled; - bool reset_on_next_read; - struct mon_cfg cfg; - - /* The value last read from the hardware. Used to detect overflow. */ - u64 prev_val; - - /* - * The value to add to the new reading to account for power management, - * and shifts to trigger the overflow interrupt. - */ - u64 correction; -}; - -struct mpam_msc_ris { - u8 ris_idx; - u64 idr; - struct mpam_props props; - bool in_reset_state; - - cpumask_t affinity; - - /* member of mpam_component:ris */ - struct list_head comp_list; - - /* member of mpam_msc:ris */ - struct list_head msc_list; - - /* parents: */ - struct mpam_msc *msc; - struct mpam_component *comp; - - /* msmon mbwu configuration is preserved over reset */ - struct msmon_mbwu_state *mbwu_state; -}; - -struct mpam_resctrl_dom { - struct mpam_component *comp; - struct rdt_domain resctrl_dom; - - u32 mbm_local_evt_cfg; -}; - -struct mpam_resctrl_res { - struct mpam_class *class; - struct rdt_resource resctrl_res; -}; - -static inline int mpam_alloc_csu_mon(struct mpam_class *class) -{ - struct mpam_props *cprops = &class->props; - - if (!mpam_has_feature(mpam_feat_msmon_csu, cprops)) - return -EOPNOTSUPP; - - return ida_alloc_range(&class->ida_csu_mon, 0, cprops->num_csu_mon - 1, - GFP_KERNEL); -} - -static inline void mpam_free_csu_mon(struct mpam_class *class, int csu_mon) -{ - ida_free(&class->ida_csu_mon, csu_mon); -} - -static inline int mpam_alloc_mbwu_mon(struct mpam_class *class) -{ - struct mpam_props *cprops = &class->props; - - if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) - return -EOPNOTSUPP; - - return ida_alloc_range(&class->ida_mbwu_mon, 0, - cprops->num_mbwu_mon - 1, GFP_KERNEL); -} - -static inline void mpam_free_mbwu_mon(struct mpam_class *class, int mbwu_mon) -{ - ida_free(&class->ida_mbwu_mon, mbwu_mon); -} - -/* List of all classes */ -extern struct list_head mpam_classes; -extern struct srcu_struct mpam_srcu; - -/* System wide partid/pmg values */ -extern u16 mpam_partid_max; -extern u8 mpam_pmg_max; - -/* Scheduled work callback to enable mpam once all MSC have been probed */ -void mpam_enable(struct work_struct *work); -void mpam_disable(struct work_struct *work); - -void mpam_reset_class(struct mpam_class *class); - -int mpam_apply_config(struct mpam_component *comp, u16 partid, - struct mpam_config *cfg); - -int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx, - enum mpam_device_features, u64 *val); -void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx); -void mpam_msmon_reset_all_mbwu(struct mpam_component *comp); - -int mpam_resctrl_online_cpu(unsigned int cpu); -int mpam_resctrl_offline_cpu(unsigned int cpu); - -int mpam_resctrl_setup(void); -void mpam_resctrl_exit(void); - -/* - * MPAM MSCs have the following register layout. See: - * Arm Architecture Reference Manual Supplement - Memory System Resource - * Partitioning and Monitoring (MPAM), for Armv8-A. DDI 0598A.a - */ -#define MPAM_ARCHITECTURE_V1 0x10 -#define MPAM_MIN_MMIO_SIZE 0x3000 - -/* Memory mapped control pages: */ -/* ID Register offsets in the memory mapped page */ -#define MPAMF_IDR 0x0000 /* features id register */ -#define MPAMF_MSMON_IDR 0x0080 /* performance monitoring features */ -#define MPAMF_IMPL_IDR 0x0028 /* imp-def partitioning */ -#define MPAMF_CPOR_IDR 0x0030 /* cache-portion partitioning */ -#define MPAMF_CCAP_IDR 0x0038 /* cache-capacity partitioning */ -#define MPAMF_MBW_IDR 0x0040 /* mem-bw partitioning */ -#define MPAMF_PRI_IDR 0x0048 /* priority partitioning */ -#define MPAMF_CSUMON_IDR 0x0088 /* cache-usage monitor */ -#define MPAMF_MBWUMON_IDR 0x0090 /* mem-bw usage monitor */ -#define MPAMF_PARTID_NRW_IDR 0x0050 /* partid-narrowing */ -#define MPAMF_IIDR 0x0018 /* implementer id register */ -#define MPAMF_AIDR 0x0020 /* architectural id register */ - -/* Configuration and Status Register offsets in the memory mapped page */ -#define MPAMCFG_PART_SEL 0x0100 /* partid to configure: */ -#define MPAMCFG_CPBM 0x1000 /* cache-portion config */ -#define MPAMCFG_CMAX 0x0108 /* cache-capacity config */ -#define MPAMCFG_MBW_MIN 0x0200 /* min mem-bw config */ -#define MPAMCFG_MBW_MAX 0x0208 /* max mem-bw config */ -#define MPAMCFG_MBW_WINWD 0x0220 /* mem-bw accounting window config */ -#define MPAMCFG_MBW_PBM 0x2000 /* mem-bw portion bitmap config */ -#define MPAMCFG_PRI 0x0400 /* priority partitioning config */ -#define MPAMCFG_MBW_PROP 0x0500 /* mem-bw stride config */ -#define MPAMCFG_INTPARTID 0x0600 /* partid-narrowing config */ - -#define MSMON_CFG_MON_SEL 0x0800 /* monitor selector */ -#define MSMON_CFG_CSU_FLT 0x0810 /* cache-usage monitor filter */ -#define MSMON_CFG_CSU_CTL 0x0818 /* cache-usage monitor config */ -#define MSMON_CFG_MBWU_FLT 0x0820 /* mem-bw monitor filter */ -#define MSMON_CFG_MBWU_CTL 0x0828 /* mem-bw monitor config */ -#define MSMON_CSU 0x0840 /* current cache-usage */ -#define MSMON_CSU_CAPTURE 0x0848 /* last cache-usage value captured */ -#define MSMON_MBWU 0x0860 /* current mem-bw usage value */ -#define MSMON_MBWU_CAPTURE 0x0868 /* last mem-bw value captured */ -#define MSMON_MBWU_L 0x0880 /* current long mem-bw usage value */ -#define MSMON_MBWU_CAPTURE_L 0x0890 /* last long mem-bw value captured */ -#define MSMON_CAPT_EVNT 0x0808 /* signal a capture event */ -#define MPAMF_ESR 0x00F8 /* error status register */ -#define MPAMF_ECR 0x00F0 /* error control register */ - -/* MPAMF_IDR - MPAM features ID register */ -#define MPAMF_IDR_PARTID_MAX GENMASK(15, 0) -#define MPAMF_IDR_PMG_MAX GENMASK(23, 16) -#define MPAMF_IDR_HAS_CCAP_PART BIT(24) -#define MPAMF_IDR_HAS_CPOR_PART BIT(25) -#define MPAMF_IDR_HAS_MBW_PART BIT(26) -#define MPAMF_IDR_HAS_PRI_PART BIT(27) -#define MPAMF_IDR_HAS_EXT BIT(28) -#define MPAMF_IDR_HAS_IMPL_IDR BIT(29) -#define MPAMF_IDR_HAS_MSMON BIT(30) -#define MPAMF_IDR_HAS_PARTID_NRW BIT(31) -#define MPAMF_IDR_HAS_RIS BIT(32) -#define MPAMF_IDR_HAS_EXT_ESR BIT(38) -#define MPAMF_IDR_HAS_ESR BIT(39) -#define MPAMF_IDR_RIS_MAX GENMASK(59, 56) - - -/* MPAMF_MSMON_IDR - MPAM performance monitoring ID register */ -#define MPAMF_MSMON_IDR_MSMON_CSU BIT(16) -#define MPAMF_MSMON_IDR_MSMON_MBWU BIT(17) -#define MPAMF_MSMON_IDR_HAS_LOCAL_CAPT_EVNT BIT(31) - -/* MPAMF_CPOR_IDR - MPAM features cache portion partitioning ID register */ -#define MPAMF_CPOR_IDR_CPBM_WD GENMASK(15, 0) - -/* MPAMF_CCAP_IDR - MPAM features cache capacity partitioning ID register */ -#define MPAMF_CCAP_IDR_CMAX_WD GENMASK(5, 0) - -/* MPAMF_MBW_IDR - MPAM features memory bandwidth partitioning ID register */ -#define MPAMF_MBW_IDR_BWA_WD GENMASK(5, 0) -#define MPAMF_MBW_IDR_HAS_MIN BIT(10) -#define MPAMF_MBW_IDR_HAS_MAX BIT(11) -#define MPAMF_MBW_IDR_HAS_PBM BIT(12) -#define MPAMF_MBW_IDR_HAS_PROP BIT(13) -#define MPAMF_MBW_IDR_WINDWR BIT(14) -#define MPAMF_MBW_IDR_BWPBM_WD GENMASK(28, 16) - -/* MPAMF_PRI_IDR - MPAM features priority partitioning ID register */ -#define MPAMF_PRI_IDR_HAS_INTPRI BIT(0) -#define MPAMF_PRI_IDR_INTPRI_0_IS_LOW BIT(1) -#define MPAMF_PRI_IDR_INTPRI_WD GENMASK(9, 4) -#define MPAMF_PRI_IDR_HAS_DSPRI BIT(16) -#define MPAMF_PRI_IDR_DSPRI_0_IS_LOW BIT(17) -#define MPAMF_PRI_IDR_DSPRI_WD GENMASK(25, 20) - -/* MPAMF_CSUMON_IDR - MPAM cache storage usage monitor ID register */ -#define MPAMF_CSUMON_IDR_NUM_MON GENMASK(15, 0) -#define MPAMF_CSUMON_IDR_HAS_CAPTURE BIT(31) - -/* MPAMF_MBWUMON_IDR - MPAM memory bandwidth usage monitor ID register */ -#define MPAMF_MBWUMON_IDR_NUM_MON GENMASK(15, 0) -#define MPAMF_MBWUMON_IDR_HAS_RWBW BIT(28) -#define MPAMF_MBWUMON_IDR_LWD BIT(29) -#define MPAMF_MBWUMON_IDR_HAS_LONG BIT(30) -#define MPAMF_MBWUMON_IDR_HAS_CAPTURE BIT(31) - -/* MPAMF_PARTID_NRW_IDR - MPAM PARTID narrowing ID register */ -#define MPAMF_PARTID_NRW_IDR_INTPARTID_MAX GENMASK(15, 0) - -/* MPAMF_IIDR - MPAM implementation ID register */ -#define MPAMF_IIDR_PRODUCTID GENMASK(31, 20) -#define MPAMF_IIDR_PRODUCTID_SHIFT 20 -#define MPAMF_IIDR_VARIANT GENMASK(19, 16) -#define MPAMF_IIDR_VARIANT_SHIFT 16 -#define MPAMF_IIDR_REVISON GENMASK(15, 12) -#define MPAMF_IIDR_REVISON_SHIFT 12 -#define MPAMF_IIDR_IMPLEMENTER GENMASK(11, 0) -#define MPAMF_IIDR_IMPLEMENTER_SHIFT 0 - -/* MPAMF_AIDR - MPAM architecture ID register */ -#define MPAMF_AIDR_ARCH_MAJOR_REV GENMASK(7, 4) -#define MPAMF_AIDR_ARCH_MINOR_REV GENMASK(3, 0) - -/* MPAMCFG_PART_SEL - MPAM partition configuration selection register */ -#define MPAMCFG_PART_SEL_PARTID_SEL GENMASK(15, 0) -#define MPAMCFG_PART_SEL_INTERNAL BIT(16) -#define MPAMCFG_PART_SEL_RIS GENMASK(27, 24) - -/* MPAMCFG_CMAX - MPAM cache portion bitmap partition configuration register */ -#define MPAMCFG_CMAX_CMAX GENMASK(15, 0) - -/* - * MPAMCFG_MBW_MIN - MPAM memory minimum bandwidth partitioning configuration - * register - */ -#define MPAMCFG_MBW_MIN_MIN GENMASK(15, 0) - -/* - * MPAMCFG_MBW_MAX - MPAM memory maximum bandwidth partitioning configuration - * register - */ -#define MPAMCFG_MBW_MAX_MAX GENMASK(15, 0) -#define MPAMCFG_MBW_MAX_HARDLIM BIT(31) - -/* - * MPAMCFG_MBW_WINWD - MPAM memory bandwidth partitioning window width - * register - */ -#define MPAMCFG_MBW_WINWD_US_FRAC GENMASK(7, 0) -#define MPAMCFG_MBW_WINWD_US_INT GENMASK(23, 8) - - -/* MPAMCFG_PRI - MPAM priority partitioning configuration register */ -#define MPAMCFG_PRI_INTPRI GENMASK(15, 0) -#define MPAMCFG_PRI_DSPRI GENMASK(31, 16) - -/* - * MPAMCFG_MBW_PROP - Memory bandwidth proportional stride partitioning - * configuration register - */ -#define MPAMCFG_MBW_PROP_STRIDEM1 GENMASK(15, 0) -#define MPAMCFG_MBW_PROP_EN BIT(31) - -/* - * MPAMCFG_INTPARTID - MPAM internal partition narrowing configuration register - */ -#define MPAMCFG_INTPARTID_INTPARTID GENMASK(15, 0) -#define MPAMCFG_INTPARTID_INTERNAL BIT(16) - -/* MSMON_CFG_MON_SEL - Memory system performance monitor selection register */ -#define MSMON_CFG_MON_SEL_MON_SEL GENMASK(7, 0) -#define MSMON_CFG_MON_SEL_RIS GENMASK(27, 24) - -/* MPAMF_ESR - MPAM Error Status Register */ -#define MPAMF_ESR_PARTID_OR_MON GENMASK(15, 0) -#define MPAMF_ESR_PMG GENMASK(23, 16) -#define MPAMF_ESR_ERRCODE GENMASK(27, 24) -#define MPAMF_ESR_OVRWR BIT(31) -#define MPAMF_ESR_RIS GENMASK(35, 32) - -/* MPAMF_ECR - MPAM Error Control Register */ -#define MPAMF_ECR_INTEN BIT(0) - -/* Error conditions in accessing memory mapped registers */ -#define MPAM_ERRCODE_NONE 0 -#define MPAM_ERRCODE_PARTID_SEL_RANGE 1 -#define MPAM_ERRCODE_REQ_PARTID_RANGE 2 -#define MPAM_ERRCODE_MSMONCFG_ID_RANGE 3 -#define MPAM_ERRCODE_REQ_PMG_RANGE 4 -#define MPAM_ERRCODE_MONITOR_RANGE 5 -#define MPAM_ERRCODE_INTPARTID_RANGE 6 -#define MPAM_ERRCODE_UNEXPECTED_INTERNAL 7 - -/* - * MSMON_CFG_CSU_FLT - Memory system performance monitor configure cache storage - * usage monitor filter register - */ -#define MSMON_CFG_CSU_FLT_PARTID GENMASK(15, 0) -#define MSMON_CFG_CSU_FLT_PMG GENMASK(23, 16) - -/* - * MSMON_CFG_CSU_CTL - Memory system performance monitor configure cache storage - * usage monitor control register - * MSMON_CFG_MBWU_CTL - Memory system performance monitor configure memory - * bandwidth usage monitor control register - */ -#define MSMON_CFG_x_CTL_TYPE GENMASK(7, 0) -#define MSMON_CFG_x_CTL_MATCH_PARTID BIT(16) -#define MSMON_CFG_x_CTL_MATCH_PMG BIT(17) -#define MSMON_CFG_x_CTL_SCLEN BIT(19) -#define MSMON_CFG_x_CTL_SUBTYPE GENMASK(23, 20) -#define MSMON_CFG_x_CTL_OFLOW_FRZ BIT(24) -#define MSMON_CFG_x_CTL_OFLOW_INTR BIT(25) -#define MSMON_CFG_x_CTL_OFLOW_STATUS BIT(26) -#define MSMON_CFG_x_CTL_CAPT_RESET BIT(27) -#define MSMON_CFG_x_CTL_CAPT_EVNT GENMASK(30, 28) -#define MSMON_CFG_x_CTL_EN BIT(31) - -#define MSMON_CFG_MBWU_CTL_TYPE_MBWU 0x42 -#define MSMON_CFG_MBWU_CTL_TYPE_CSU 0x43 - -#define MSMON_CFG_MBWU_CTL_SUBTYPE_NONE 0 -#define MSMON_CFG_MBWU_CTL_SUBTYPE_READ 1 -#define MSMON_CFG_MBWU_CTL_SUBTYPE_WRITE 2 -#define MSMON_CFG_MBWU_CTL_SUBTYPE_BOTH 3 - -#define MSMON_CFG_MBWU_CTL_SUBTYPE_MAX 3 -#define MSMON_CFG_MBWU_CTL_SUBTYPE_MASK 0x3 - -/* - * MSMON_CFG_MBWU_FLT - Memory system performance monitor configure memory - * bandwidth usage monitor filter register - */ -#define MSMON_CFG_MBWU_FLT_PARTID GENMASK(15, 0) -#define MSMON_CFG_MBWU_FLT_PMG GENMASK(23, 16) -#define MSMON_CFG_MBWU_FLT_RWBW GENMASK(31, 30) - -/* - * MSMON_CSU - Memory system performance monitor cache storage usage monitor - * register - * MSMON_CSU_CAPTURE - Memory system performance monitor cache storage usage - * capture register - * MSMON_MBWU - Memory system performance monitor memory bandwidth usage - * monitor register - * MSMON_MBWU_CAPTURE - Memory system performance monitor memory bandwidth usage - * capture register - */ -#define MSMON___VALUE GENMASK(30, 0) -#define MSMON___NRDY BIT(31) -#define MSMON___NRDY_L BIT(63) -#define MSMON___L_VALUE GENMASK(43, 0) -#define MSMON___LWD_VALUE GENMASK(62, 0) - -/* - * MSMON_CAPT_EVNT - Memory system performance monitoring capture event - * generation register - */ -#define MSMON_CAPT_EVNT_NOW BIT(0) - -/* Used for PTG Yitian710 specific MB monitoring feature */ -#define MBWU_MASK GENMASK(23, 0) -#define MBWU_WINWD_MAX GENMASK(22, 0) -#define MBWU_GET(v) ((v) & MBWU_MASK) -#define MPAMF_CUST_MBWC_OFFSET 0x08 -#define MPAMF_CUST_WINDW_OFFSET 0x0C - -#endif /* MPAM_INTERNAL_H */ diff --git a/drivers/platform/mpam/mpam_resctrl.c b/drivers/platform/mpam/mpam_resctrl.c deleted file mode 100644 index ee1a66a6c09924426f87e0350c3c555ff6682c39..0000000000000000000000000000000000000000 --- a/drivers/platform/mpam/mpam_resctrl.c +++ /dev/null @@ -1,1276 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2021 Arm Ltd. - -#define pr_fmt(fmt) "mpam: resctrl: " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "mpam_internal.h" - -u64 mpam_resctrl_default_group; - -DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); - -/* - * The classes we've picked to map to resctrl resources. - * Class pointer may be NULL. - */ -static struct mpam_resctrl_res mpam_resctrl_exports[RDT_NUM_RESOURCES]; - -static bool exposed_alloc_capable; -static bool exposed_mon_capable; -static struct mpam_class *mbm_local_class; -static struct mpam_class *mbm_total_class; -static struct mpam_class *mbm_bps_class; - -/* - * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM1_EL1. - * This applies globally to all traffic the CPU generates. - */ -static bool cdp_enabled; - -/* - * If resctrl_init() succeeded, resctrl_exit() can be used to remove support - * for the filesystem in the event of an error. - */ -static bool resctrl_enabled; - -/* - * mpam_resctrl_pick_caches() needs to know the size of the caches. cacheinfo - * populates this from a device_initcall(). mpam_resctrl_setup() must wait. - */ -static bool cacheinfo_ready; -static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready); - -/* A dummy mon context to use when the monitors were allocated up front */ -u32 __mon_is_rmid_idx = USE_RMID_IDX; -void *mon_is_rmid_idx = &__mon_is_rmid_idx; - -bool resctrl_arch_alloc_capable(void) -{ - return exposed_alloc_capable; -} - -bool resctrl_arch_mon_capable(void) -{ - return exposed_mon_capable; -} - -bool resctrl_arch_is_mbm_local_enabled(void) -{ - return mbm_local_class; -} - -bool resctrl_arch_is_mbm_total_enabled(void) -{ - return mbm_total_class; -} - -bool resctrl_arch_is_mbm_bps_enabled(void) -{ - return mbm_bps_class; -} - -bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) -{ - switch (rid) { - case RDT_RESOURCE_L2: - case RDT_RESOURCE_L3: - return cdp_enabled; - case RDT_RESOURCE_MBA: - default: - /* - * x86's MBA control doesn't support CDP, so user-space doesn't - * expect it. - */ - return false; - } -} - -int resctrl_arch_set_cdp_enabled(enum resctrl_res_level ignored, bool enable) -{ - u64 regval; - u32 partid, partid_i, partid_d; - - cdp_enabled = enable; - - partid = RESCTRL_RESERVED_CLOSID; - - if (enable) { - partid_d = resctrl_get_config_index(partid, CDP_CODE); - partid_i = resctrl_get_config_index(partid, CDP_DATA); - regval = FIELD_PREP(MPAM_SYSREG_PARTID_D, partid_d) | - FIELD_PREP(MPAM_SYSREG_PARTID_I, partid_i); - - } else { - regval = FIELD_PREP(MPAM_SYSREG_PARTID_D, partid) | - FIELD_PREP(MPAM_SYSREG_PARTID_I, partid); - } - - WRITE_ONCE(mpam_resctrl_default_group, regval); - - return 0; -} - -static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid) -{ - return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid); -} - -/* - * MSC may raise an error interrupt if it sees an out or range partid/pmg, - * and go on to truncate the value. Regardless of what the hardware supports, - * only the system wide safe value is safe to use. - */ -u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored) -{ - return mpam_partid_max + 1; -} - -u32 resctrl_arch_system_num_rmid_idx(void) -{ - u8 closid_shift = fls(mpam_pmg_max); - u32 num_partid = resctrl_arch_get_num_closid(NULL); - - return num_partid << closid_shift; -} - -u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid) -{ - u8 closid_shift = fls(mpam_pmg_max); - - if (WARN_ON_ONCE(closid_shift > 8)) - closid_shift = 8; - - return (closid << closid_shift) | rmid; -} - -void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid) -{ - u8 closid_shift = fls(mpam_pmg_max); - u32 pmg_mask = ~(~0 << closid_shift); - - if (WARN_ON_ONCE(closid_shift > 8)) - closid_shift = 8; - - *closid = idx >> closid_shift; - *rmid = idx & pmg_mask; -} - -void resctrl_arch_sched_in(struct task_struct *tsk) -{ - lockdep_assert_preemption_disabled(); - - mpam_thread_switch(tsk); -} - -void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 pmg) -{ - if (WARN_ON_ONCE(closid > U16_MAX) || WARN_ON_ONCE(pmg > U8_MAX)) - return; - - if (!cdp_enabled) { - mpam_set_cpu_defaults(cpu, closid, closid, pmg, pmg); - } else { - /* - * When CDP is enabled, resctrl halves the closid range and we - * use odd/even partid for one closid. - */ - u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); - u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); - - mpam_set_cpu_defaults(cpu, partid_d, partid_i, pmg, pmg); - } -} - -void resctrl_arch_sync_cpu_defaults(void *info) -{ - struct resctrl_cpu_sync *r = info; - - lockdep_assert_preemption_disabled(); - - if (r) { - resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(), - r->closid, r->rmid); - } - - resctrl_arch_sched_in(current); -} - -void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid) -{ - - - if (WARN_ON_ONCE(closid > U16_MAX) || WARN_ON_ONCE(rmid > U8_MAX)) - return; - - if (!cdp_enabled) { - mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid); - } else { - u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); - u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); - - mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid); - } -} - -bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid) -{ - u64 regval = mpam_get_regval(tsk); - u32 tsk_closid = FIELD_GET(MPAM_SYSREG_PARTID_D, regval); - - if (cdp_enabled) - tsk_closid >>= 1; - - return tsk_closid == closid; -} - -/* The task's pmg is not unique, the partid must be considered too */ -bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid) -{ - u64 regval = mpam_get_regval(tsk); - u32 tsk_closid = FIELD_GET(MPAM_SYSREG_PARTID_D, regval); - u32 tsk_rmid = FIELD_GET(MPAM_SYSREG_PMG_D, regval); - - if (cdp_enabled) - tsk_closid >>= 1; - - return (tsk_closid == closid) && (tsk_rmid == rmid); -} - -struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) -{ - if (l >= RDT_NUM_RESOURCES) - return NULL; - - return &mpam_resctrl_exports[l].resctrl_res; -} - -static void *resctrl_arch_mon_ctx_alloc_no_wait(struct rdt_resource *r, - int evtid) -{ - struct mpam_resctrl_res *res; - u32 *ret = kmalloc(sizeof(*ret), GFP_KERNEL); - - if (!ret) - return ERR_PTR(-ENOMEM); - - switch (evtid) { - case QOS_L3_OCCUP_EVENT_ID: - res = container_of(r, struct mpam_resctrl_res, resctrl_res); - - *ret = mpam_alloc_csu_mon(res->class); - return ret; - case QOS_L3_MBM_LOCAL_EVENT_ID: - case QOS_L3_MBM_TOTAL_EVENT_ID: - return mon_is_rmid_idx; - case QOS_MC_MBM_BPS_EVENT_ID: - if (mpam_current_machine == MPAM_YITIAN710) - return mon_is_rmid_idx; - return ERR_PTR(-EOPNOTSUPP); - } - - return ERR_PTR(-EOPNOTSUPP); -} - -void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, int evtid) -{ - DEFINE_WAIT(wait); - void *ret; - - might_sleep(); - - do { - prepare_to_wait(&resctrl_mon_ctx_waiters, &wait, - TASK_INTERRUPTIBLE); - ret = resctrl_arch_mon_ctx_alloc_no_wait(r, evtid); - if (PTR_ERR(ret) == -ENOSPC) - schedule(); - } while (PTR_ERR(ret) == -ENOSPC && !signal_pending(current)); - finish_wait(&resctrl_mon_ctx_waiters, &wait); - - return ret; -} - -void resctrl_arch_mon_ctx_free(struct rdt_resource *r, int evtid, - void *arch_mon_ctx) -{ - struct mpam_resctrl_res *res; - u32 mon = *(u32 *)arch_mon_ctx; - - if (mon == USE_RMID_IDX) - return; - kfree(arch_mon_ctx); - arch_mon_ctx = NULL; - - res = container_of(r, struct mpam_resctrl_res, resctrl_res); - - switch (evtid) { - case QOS_L3_OCCUP_EVENT_ID: - mpam_free_csu_mon(res->class, mon); - wake_up(&resctrl_mon_ctx_waiters); - return; - case QOS_L3_MBM_TOTAL_EVENT_ID: - case QOS_L3_MBM_LOCAL_EVENT_ID: - case QOS_MC_MBM_BPS_EVENT_ID: - return; - } -} - -static enum mon_filter_options resctrl_evt_config_to_mpam(u32 local_evt_cfg) -{ - switch (local_evt_cfg) { - case READS_TO_LOCAL_MEM: - return COUNT_READ; - case NON_TEMP_WRITE_TO_LOCAL_MEM: - return COUNT_WRITE; - default: - return COUNT_BOTH; - } -} - -int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, - u32 closid, u32 rmid, enum resctrl_event_id eventid, - u64 *val, void *arch_mon_ctx) -{ - int err; - u64 cdp_val; - struct mon_cfg cfg; - struct mpam_resctrl_dom *dom; - u32 mon = *(u32 *)arch_mon_ctx; - enum mpam_device_features type; - - resctrl_arch_rmid_read_context_check(); - - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - - switch (eventid) { - case QOS_L3_OCCUP_EVENT_ID: - type = mpam_feat_msmon_csu; - break; - case QOS_L3_MBM_LOCAL_EVENT_ID: - case QOS_L3_MBM_TOTAL_EVENT_ID: - type = mpam_feat_msmon_mbwu; - break; - case QOS_MC_MBM_BPS_EVENT_ID: - if (mpam_current_machine == MPAM_YITIAN710) - type = mpam_feat_impl_msmon_mbwu; - break; - default: - return -EINVAL; - } - - if (mon == USE_RMID_IDX) - cfg.mon = resctrl_arch_rmid_idx_encode(closid, rmid); - else - cfg.mon = mon; - - cfg.match_pmg = true; - cfg.pmg = rmid; - cfg.opts = resctrl_evt_config_to_mpam(dom->mbm_local_evt_cfg); - - if (cdp_enabled) { - cfg.partid = closid << 1; - err = mpam_msmon_read(dom->comp, &cfg, type, val); - if (err) - return err; - - cfg.partid += 1; - err = mpam_msmon_read(dom->comp, &cfg, type, &cdp_val); - if (!err) - *val += cdp_val; - } else { - cfg.partid = closid; - err = mpam_msmon_read(dom->comp, &cfg, type, val); - } - - return err; -} - -void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, - u32 closid, u32 rmid, enum resctrl_event_id eventid) -{ - struct mon_cfg cfg; - struct mpam_resctrl_dom *dom; - - if (eventid != QOS_L3_MBM_LOCAL_EVENT_ID) - return; - - cfg.mon = resctrl_arch_rmid_idx_encode(closid, rmid); - cfg.match_pmg = true; - cfg.pmg = rmid; - - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - - if (cdp_enabled) { - cfg.partid = closid << 1; - mpam_msmon_reset_mbwu(dom->comp, &cfg); - - cfg.partid += 1; - mpam_msmon_reset_mbwu(dom->comp, &cfg); - } else { - cfg.partid = closid; - mpam_msmon_reset_mbwu(dom->comp, &cfg); - } -} - -/* - * The rmid realloc threshold should be for the smallest cache exposed to - * resctrl. - */ -static void update_rmid_limits(unsigned int size) -{ - u32 num_unique_pmg = resctrl_arch_system_num_rmid_idx(); - - if (WARN_ON_ONCE(!size)) - return; - - if (resctrl_rmid_realloc_limit && size > resctrl_rmid_realloc_limit) - return; - - resctrl_rmid_realloc_limit = size; - resctrl_rmid_realloc_threshold = size / num_unique_pmg; -} - -static bool cache_has_usable_cpor(struct mpam_class *class) -{ - struct mpam_props *cprops = &class->props; - - if (!mpam_has_feature(mpam_feat_cpor_part, cprops)) - return false; - - /* TODO: Scaling is not yet supported */ - return (class->props.cpbm_wd <= RESCTRL_MAX_CBM); -} - -static bool cache_has_usable_csu(struct mpam_class *class) -{ - struct mpam_props *cprops; - - if (!class) - return false; - - cprops = &class->props; - - if (!mpam_has_feature(mpam_feat_msmon_csu, cprops)) - return false; - - /* - * CSU counters settle on the value, so we can get away with - * having only one. - */ - if (!cprops->num_csu_mon) - return false; - - return (mpam_partid_max > 1) || (mpam_pmg_max != 0); -} - -bool resctrl_arch_is_llc_occupancy_enabled(void) -{ - return cache_has_usable_csu(mpam_resctrl_exports[RDT_RESOURCE_L3].class); -} - -static bool class_has_usable_mbwu(struct mpam_class *class) -{ - struct mpam_props *cprops = &class->props; - - if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) - return false; - - /* - * resctrl expects the bandwidth counters to be free running, - * which means we need as many monitors as resctrl has - * control/monitor groups. - */ - if (cprops->num_mbwu_mon < resctrl_arch_system_num_rmid_idx()) - return false; - - return (mpam_partid_max > 1) || (mpam_pmg_max != 0); -} - -static bool class_has_usable_impl_mbwu(struct mpam_class *class) -{ - struct mpam_props *cprops = &class->props; - - if (!mpam_has_feature(mpam_feat_impl_msmon_mbwu, cprops)) - return false; - - return true; -} - -static bool mba_class_use_mbw_part(struct mpam_props *cprops) -{ - /* TODO: Scaling is not yet supported */ - return (mpam_has_feature(mpam_feat_mbw_part, cprops) && - cprops->mbw_pbm_bits < MAX_MBA_BW); -} - -static bool class_has_usable_mba(struct mpam_props *cprops) -{ - if (mba_class_use_mbw_part(cprops) || - mpam_has_feature(mpam_feat_mbw_max, cprops)) - return true; - - return false; -} - -/* - * Calculate the percentage change from each implemented bit in the control - * This can return 0 when BWA_WD is greater than 6. (100 / (1<<7) == 0) - */ -static u32 get_mba_granularity(struct mpam_props *cprops) -{ - if (mba_class_use_mbw_part(cprops)) { - return MAX_MBA_BW / cprops->mbw_pbm_bits; - } else if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { - /* - * bwa_wd is the number of bits implemented in the 0.xxx - * fixed point fraction. 1 bit is 50%, 2 is 25% etc. - */ - return max_t(u32, 1, (MAX_MBA_BW / BIT(cprops->bwa_wd))); - } - - return 0; -} - -static u32 mbw_pbm_to_percent(unsigned long mbw_pbm, struct mpam_props *cprops) -{ - u32 bit, result = 0, granularity = get_mba_granularity(cprops); - - for_each_set_bit(bit, &mbw_pbm, cprops->mbw_pbm_bits % 32) { - result += granularity; - } - - return result; -} - -static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops) -{ - u8 bit; - u32 divisor = 2, value = 0; - - for (bit = 15; bit; bit--) { - if (mbw_max & BIT(bit)) - /* - * Left shift by 16 bits to preserve the precision of - * the division operation. - */ - value += (MAX_MBA_BW << 16) / divisor; - divisor <<= 1; - } - - /* Use the upper bound of the fixed-point fraction. */ - value = (value + (MAX_MBA_BW << (16 - cprops->bwa_wd))) >> 16; - - return value; -} - -static u32 percent_to_mbw_pbm(u8 pc, struct mpam_props *cprops) -{ - u32 granularity = get_mba_granularity(cprops); - u8 num_bits = pc / granularity; - - if (!num_bits) - return 0; - - /* TODO: pick bits at random to avoid contention */ - return (1 << num_bits) - 1; -} - -static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) -{ - u8 bit; - u32 granularity, pc_ls, divisor = 2, value = 0; - - if (WARN_ON_ONCE(cprops->bwa_wd > 15)) - return MAX_MBA_BW; - - /* Set the pc value to be a multiple of granularity. */ - granularity = get_mba_granularity(cprops); - pc = roundup(pc, (u8) granularity); - if (pc > 100) - pc = 100; - - /* - * Left shift by 16 bits to preserve the precision of the division - * operation. - */ - pc_ls = (u32) pc << 16; - - for (bit = 15; bit; bit--) { - if (pc_ls >= (MAX_MBA_BW << 16) / divisor) { - pc_ls -= (MAX_MBA_BW << 16) / divisor; - value |= BIT(bit); - } - divisor <<= 1; - - if (!pc_ls || !((MAX_MBA_BW << 16) / divisor)) - break; - } - - value &= GENMASK(15, 15 - cprops->bwa_wd + 1); - - return value; -} - -/* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */ -static void mpam_resctrl_pick_caches(void) -{ - int idx; - unsigned int cache_size; - struct mpam_class *class; - struct mpam_resctrl_res *res; - - lockdep_assert_cpus_held(); - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(class, &mpam_classes, classes_list) { - struct mpam_props *cprops = &class->props; - bool has_cpor = cache_has_usable_cpor(class); - - if (class->type != MPAM_CLASS_CACHE) { - pr_debug("pick_caches: Class is not a cache\n"); - continue; - } - - if (class->level != 2 && class->level != 3) { - pr_debug("pick_caches: not L2 or L3\n"); - continue; - } - - if (class->level == 2 && !has_cpor) { - pr_debug("pick_caches: L2 missing CPOR\n"); - continue; - } else if (!has_cpor && !cache_has_usable_csu(class)) { - pr_debug("pick_caches: Cache misses CPOR and CSU\n"); - continue; - } - - if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { - pr_debug("pick_caches: Class has missing CPUs\n"); - continue; - } - - /* Assume cache levels are the same size for all CPUs... */ - cache_size = get_cpu_cacheinfo_size(smp_processor_id(), class->level); - if (!cache_size) { - pr_debug("pick_caches: Could not read cache size\n"); - continue; - } - - if (mpam_has_feature(mpam_feat_msmon_csu, cprops)) - update_rmid_limits(cache_size); - - if (class->level == 2) { - res = &mpam_resctrl_exports[RDT_RESOURCE_L2]; - res->resctrl_res.name = "L2"; - } else { - res = &mpam_resctrl_exports[RDT_RESOURCE_L3]; - res->resctrl_res.name = "L3"; - } - res->class = class; - } - srcu_read_unlock(&mpam_srcu, idx); -} - -static void mpam_resctrl_pick_mba(void) -{ - struct mpam_class *class, *candidate_class = NULL; - struct mpam_resctrl_res *res; - int idx; - - lockdep_assert_cpus_held(); - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(class, &mpam_classes, classes_list) { - struct mpam_props *cprops = &class->props; - - if (class->level < 3) - continue; - - if (!class_has_usable_mba(cprops)) - continue; - - if (!cpumask_equal(&class->affinity, cpu_possible_mask)) - continue; - - /* - * mba_sc reads the mbm_local counter, and waggles the MBA controls. - * mbm_local is implicitly part of the L3, pick a resouce to be MBA - * that as close as possible to the L3. - */ - if (!candidate_class || class->level < candidate_class->level) - candidate_class = class; - } - srcu_read_unlock(&mpam_srcu, idx); - - if (candidate_class) { - res = &mpam_resctrl_exports[RDT_RESOURCE_MBA]; - res->class = candidate_class; - res->resctrl_res.name = "MB"; - } -} - -bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt) -{ - struct mpam_props *cprops; - - switch (evt) { - case QOS_L3_MBM_LOCAL_EVENT_ID: - if (!mbm_local_class) - return false; - cprops = &mbm_local_class->props; - - return mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, cprops); - default: - return false; - } -} - -u32 resctrl_arch_event_config_get(void *info, enum resctrl_event_id eventid) -{ - struct mpam_resctrl_dom *dom; - struct resctrl_mon_config_info *mon_info = info; - - dom = container_of(mon_info->d, struct mpam_resctrl_dom, resctrl_dom); - mon_info->mon_config = dom->mbm_local_evt_cfg & MAX_EVT_CONFIG_BITS; - - return mon_info->mon_config; -} - -void resctrl_arch_event_config_set(void *info) -{ - struct mpam_resctrl_dom *dom; - struct resctrl_mon_config_info *mon_info = info; - - if (mon_info->mon_config & ~MPAM_RESTRL_EVT_CONFIG_VALID) { - mon_info->err = -EOPNOTSUPP; - return; - } - - dom = container_of(mon_info->d, struct mpam_resctrl_dom, resctrl_dom); - dom->mbm_local_evt_cfg = mon_info->mon_config & MPAM_RESTRL_EVT_CONFIG_VALID; -} - -void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_domain *d) -{ - struct mpam_resctrl_dom *dom; - - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - dom->mbm_local_evt_cfg = MPAM_RESTRL_EVT_CONFIG_VALID; - mpam_msmon_reset_all_mbwu(dom->comp); -} - -static int mpam_resctrl_resource_init(struct mpam_resctrl_res *res) -{ - struct mpam_class *class = res->class; - struct rdt_resource *r = &res->resctrl_res; - bool has_mbwu = class_has_usable_mbwu(class); - - /* Is this one of the two well-known caches? */ - if (res->resctrl_res.rid == RDT_RESOURCE_L2 || - res->resctrl_res.rid == RDT_RESOURCE_L3) { - bool has_csu = cache_has_usable_csu(class); - - r->cache_level = class->level; - - /* TODO: Scaling is not yet supported */ - r->cache.cbm_len = class->props.cpbm_wd; - r->cache.arch_has_sparse_bitmasks = true; - - /* mpam_devices will reject empty bitmaps */ - r->cache.min_cbm_bits = 1; - - /* TODO: kill these properties off as they are derivatives */ - r->format_str = "%d=%0*x"; - r->fflags = RFTYPE_RES_CACHE; - r->default_ctrl = BIT_MASK(class->props.cpbm_wd) - 1; - r->data_width = (class->props.cpbm_wd + 3) / 4; - - /* - * Which bits are shared with other ...things... - * Unknown devices use partid-0 which uses all the bitmap - * fields. Until we configured the SMMU and GIC not to do this - * 'all the bits' is the correct answer here. - */ - r->cache.shareable_bits = r->default_ctrl; - - if (mpam_has_feature(mpam_feat_cpor_part, &class->props)) { - r->alloc_capable = true; - exposed_alloc_capable = true; - } - - /* - * MBWU counters may be 'local' or 'total' depending on where - * they are in the topology. Counters on caches are assumed to - * be local. If it's on the memory controller, its assumed to - * be global. - */ - if (has_mbwu && class->level >= 3) { - mbm_local_class = class; - r->mon_capable = true; - } - - /* - * CSU counters only make sense on a cache. The file is called - * llc_occupancy, but its expected to the on the L3. - */ - if (has_csu && class->type == MPAM_CLASS_CACHE && - class->level == 3) { - r->mon_capable = true; - } - } else if (res->resctrl_res.rid == RDT_RESOURCE_MBA) { - struct mpam_props *cprops = &class->props; - - /* TODO: kill these properties off as they are derivatives */ - r->format_str = "%d=%*u"; - r->fflags = RFTYPE_RES_MB; - r->default_ctrl = MAX_MBA_BW; - r->data_width = 3; - - r->membw.delay_linear = true; - r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; - r->membw.bw_gran = get_mba_granularity(cprops); - r->membw.min_bw = r->membw.bw_gran; - - /* Round up to at least 1% */ - if (!r->membw.bw_gran) - r->membw.bw_gran = 1; - - if (class_has_usable_mba(cprops)) { - r->alloc_capable = true; - exposed_alloc_capable = true; - } - - if (has_mbwu && class->type == MPAM_CLASS_MEMORY) { - mbm_total_class = class; - r->mon_capable = true; - } else if (class_has_usable_impl_mbwu(class)) { - r->mon_capable = true; - if (mpam_current_machine == MPAM_YITIAN710) - mbm_bps_class = class; - } - } - - if (r->mon_capable) { - exposed_mon_capable = true; - - /* - * Unfortunately, num_rmid doesn't mean anything for - * mpam, and its exposed to user-space! - * num-rmid is supposed to mean the number of groups - * that can be created, both control or monitor groups. - * For mpam, each control group has its own pmg/rmid - * space. - */ - r->mon.num_rmid = 1; - } - - return 0; -} - -int mpam_resctrl_setup(void) -{ - int err = 0; - struct mpam_resctrl_res *res; - enum resctrl_res_level i; - - wait_event(wait_cacheinfo_ready, cacheinfo_ready); - - cpus_read_lock(); - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - res = &mpam_resctrl_exports[i]; - INIT_LIST_HEAD(&res->resctrl_res.domains); - INIT_LIST_HEAD(&res->resctrl_res.mon.evt_list); - res->resctrl_res.rid = i; - } - - mpam_resctrl_pick_caches(); - mpam_resctrl_pick_mba(); - /* TODO: mpam_resctrl_pick_counters(); */ - - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - res = &mpam_resctrl_exports[i]; - if (!res->class) - continue; // dummy resource - - err = mpam_resctrl_resource_init(res); - if (err) - break; - } - cpus_read_unlock(); - - if (!err && !exposed_alloc_capable && !exposed_mon_capable) - err = -EOPNOTSUPP; - - if (!err) { - if (!is_power_of_2(mpam_pmg_max + 1)) { - /* - * If not all the partid*pmg values are valid indexes, - * resctrl may allocate pmg that don't exist. This - * should cause an error interrupt. - */ - pr_warn("Number of PMG is not a power of 2! resctrl may misbehave"); - } - - err = resctrl_init(); - if (!err) - WRITE_ONCE(resctrl_enabled, true); - } - - return err; -} - -void mpam_resctrl_exit(void) -{ - if (!READ_ONCE(resctrl_enabled)) - return; - - WRITE_ONCE(resctrl_enabled, false); - resctrl_exit(); -} - -u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d, - u32 closid, enum resctrl_conf_type type) -{ - u32 partid; - struct mpam_config *cfg; - struct mpam_props *cprops; - struct mpam_resctrl_res *res; - struct mpam_resctrl_dom *dom; - enum mpam_device_features configured_by; - - lockdep_assert_cpus_held(); - - if (!mpam_is_enabled()) - return r->default_ctrl; - - res = container_of(r, struct mpam_resctrl_res, resctrl_res); - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - cprops = &res->class->props; - - if (mpam_resctrl_hide_cdp(r->rid)) - partid = resctrl_get_config_index(closid, CDP_CODE); - else - partid = resctrl_get_config_index(closid, type); - cfg = &dom->comp->cfg[partid]; - - switch (r->rid) { - case RDT_RESOURCE_L2: - case RDT_RESOURCE_L3: - configured_by = mpam_feat_cpor_part; - break; - case RDT_RESOURCE_MBA: - if (mba_class_use_mbw_part(cprops)) { - configured_by = mpam_feat_mbw_part; - break; - } else if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { - configured_by = mpam_feat_mbw_max; - break; - } - fallthrough; - default: - return -EINVAL; - } - - if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r) || - !mpam_has_feature(configured_by, cfg)) - return r->default_ctrl; - - switch (configured_by) { - case mpam_feat_cpor_part: - /* TODO: Scaling is not yet supported */ - return cfg->cpbm; - case mpam_feat_mbw_part: - /* TODO: Scaling is not yet supported */ - return mbw_pbm_to_percent(cfg->mbw_pbm, cprops); - case mpam_feat_mbw_max: - return mbw_max_to_percent(cfg->mbw_max, cprops); - default: - return -EINVAL; - } -} - -int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, - u32 closid, enum resctrl_conf_type t, u32 cfg_val) -{ - int err; - u32 partid; - struct mpam_config cfg; - struct mpam_props *cprops; - struct mpam_resctrl_res *res; - struct mpam_resctrl_dom *dom; - - lockdep_assert_cpus_held(); - lockdep_assert_irqs_enabled(); - - /* - * NOTE: don't check the CPU as mpam_apply_config() doesn't care, - * and resctrl_arch_update_domains() depends on this. - */ - res = container_of(r, struct mpam_resctrl_res, resctrl_res); - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - cprops = &res->class->props; - - partid = resctrl_get_config_index(closid, t); - if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r)) - return -EINVAL; - - switch (r->rid) { - case RDT_RESOURCE_L2: - case RDT_RESOURCE_L3: - /* TODO: Scaling is not yet supported */ - cfg.cpbm = cfg_val; - mpam_set_feature(mpam_feat_cpor_part, &cfg); - break; - case RDT_RESOURCE_MBA: - if (mba_class_use_mbw_part(cprops)) { - cfg.mbw_pbm = percent_to_mbw_pbm(cfg_val, cprops); - mpam_set_feature(mpam_feat_mbw_part, &cfg); - break; - } else if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { - cfg.mbw_max = percent_to_mbw_max(cfg_val, cprops); - mpam_set_feature(mpam_feat_mbw_max, &cfg); - break; - } - fallthrough; - default: - return -EINVAL; - } - - /* - * When CDP is enabled, but the resource doesn't support it, we need to - * apply the same configuration to the other partid. - */ - if (mpam_resctrl_hide_cdp(r->rid)) { - partid = resctrl_get_config_index(closid, CDP_CODE); - err = mpam_apply_config(dom->comp, partid, &cfg); - if (err) - return err; - - partid = resctrl_get_config_index(closid, CDP_DATA); - return mpam_apply_config(dom->comp, partid, &cfg); - - } else { - return mpam_apply_config(dom->comp, partid, &cfg); - } -} - -/* TODO: this is IPI heavy */ -int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) -{ - int err = 0; - struct rdt_domain *d; - enum resctrl_conf_type t; - struct resctrl_staged_config *cfg; - - lockdep_assert_cpus_held(); - lockdep_assert_irqs_enabled(); - - list_for_each_entry(d, &r->domains, list) { - for (t = 0; t < CDP_NUM_TYPES; t++) { - cfg = &d->staged_config[t]; - if (!cfg->have_new_ctrl) - continue; - - err = resctrl_arch_update_one(r, d, closid, t, - cfg->new_ctrl); - if (err) - return err; - } - } - - return err; -} - -void resctrl_arch_reset_resources(void) -{ - int i, idx; - struct mpam_class *class; - struct mpam_resctrl_res *res; - - lockdep_assert_cpus_held(); - - if (!mpam_is_enabled()) - return; - - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - res = &mpam_resctrl_exports[i]; - - if (!res->class) - continue; // dummy resource - - if (!res->resctrl_res.alloc_capable) - continue; - - idx = srcu_read_lock(&mpam_srcu); - list_for_each_entry_rcu(class, &mpam_classes, classes_list) - mpam_reset_class(class); - srcu_read_unlock(&mpam_srcu, idx); - } -} - -static struct mpam_resctrl_dom * -mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) -{ - struct mpam_resctrl_dom *dom; - struct mpam_class *class = res->class; - struct mpam_component *comp_iter, *comp; - - comp = NULL; - list_for_each_entry(comp_iter, &class->components, class_list) { - if (cpumask_test_cpu(cpu, &comp_iter->affinity)) { - comp = comp_iter; - break; - } - } - - /* cpu with unknown exported component? */ - if (WARN_ON_ONCE(!comp)) - return ERR_PTR(-EINVAL); - - dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, cpu_to_node(cpu)); - if (!dom) - return ERR_PTR(-ENOMEM); - - dom->comp = comp; - INIT_LIST_HEAD(&dom->resctrl_dom.list); - dom->resctrl_dom.id = comp->comp_id; - dom->mbm_local_evt_cfg = MPAM_RESTRL_EVT_CONFIG_VALID; - cpumask_set_cpu(cpu, &dom->resctrl_dom.cpu_mask); - - /* TODO: this list should be sorted */ - list_add_tail(&dom->resctrl_dom.list, &res->resctrl_res.domains); - - return dom; -} - -/* Like resctrl_get_domain_from_cpu(), but for offline CPUs */ -static struct mpam_resctrl_dom * -mpam_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res) -{ - struct rdt_domain *d; - struct mpam_resctrl_dom *dom; - - lockdep_assert_cpus_held(); - - list_for_each_entry(d, &res->resctrl_res.domains, list) { - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - - if (cpumask_test_cpu(cpu, &dom->comp->affinity)) - return dom; - } - - return NULL; -} - -struct rdt_domain *resctrl_arch_find_domain(struct rdt_resource *r, int id) -{ - struct rdt_domain *d; - struct mpam_resctrl_dom *dom; - - lockdep_assert_cpus_held(); - - list_for_each_entry(d, &r->domains, list) { - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - if (dom->comp->comp_id == id) - return &dom->resctrl_dom; - } - - return NULL; -} - -int mpam_resctrl_online_cpu(unsigned int cpu) -{ - int i, err; - struct mpam_resctrl_dom *dom; - struct mpam_resctrl_res *res; - struct rdt_resource *r; - - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - res = &mpam_resctrl_exports[i]; - r = &res->resctrl_res; - - if (!res->class) - continue; // dummy_resource; - - dom = mpam_get_domain_from_cpu(cpu, res); - if (dom) { - cpumask_set_cpu(cpu, &dom->resctrl_dom.cpu_mask); - continue; - } - - dom = mpam_resctrl_alloc_domain(cpu, res); - if (IS_ERR(dom)) - return PTR_ERR(dom); - r->rdt_domain_list[i] = &dom->resctrl_dom; - err = resctrl_online_domain(&res->resctrl_res, &dom->resctrl_dom); - if (err) - return err; - } - - resctrl_online_cpu(cpu); - return 0; -} - -int mpam_resctrl_offline_cpu(unsigned int cpu) -{ - int i; - struct rdt_domain *d; - struct mpam_resctrl_res *res; - struct mpam_resctrl_dom *dom; - struct rdt_resource *r; - - resctrl_offline_cpu(cpu); - - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - res = &mpam_resctrl_exports[i]; - r = &res->resctrl_res; - - if (!res->class) - continue; // dummy resource - - d = resctrl_get_domain_from_cpu(cpu, &res->resctrl_res); - dom = container_of(d, struct mpam_resctrl_dom, resctrl_dom); - - /* The last one standing was ahead of us... */ - if (WARN_ON_ONCE(!d)) - continue; - - cpumask_clear_cpu(cpu, &d->cpu_mask); - - if (!cpumask_empty(&d->cpu_mask)) - continue; - - resctrl_offline_domain(&res->resctrl_res, &dom->resctrl_dom); - list_del(&d->list); - - r->rdt_domain_list[i] = NULL; - kfree(dom); - } - - return 0; -} - -static int __init __cacheinfo_ready(void) -{ - cacheinfo_ready = true; - wake_up(&wait_cacheinfo_ready); - - return 0; -} -device_initcall_sync(__cacheinfo_ready); diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index f6e2a4523f7e66d4a43a8ff0d2598a5649d279c8..3a0bf6adc245b85b81ffb78f32ba81e6c1e7fbe8 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -15,9 +15,11 @@ #include #include +#include + #include "kernfs-internal.h" -static DEFINE_RWLOCK(kernfs_rename_lock); /* kn->parent and ->name */ +DEFINE_RWLOCK(kernfs_rename_lock); /* kn->parent and ->name */ /* * Don't use rename_lock to piggy back on pr_cont_buf. We don't want to * call pr_cont() while holding rename_lock. Because sometimes pr_cont() @@ -51,22 +53,14 @@ static bool kernfs_lockdep(struct kernfs_node *kn) #endif } -static int kernfs_name_locked(struct kernfs_node *kn, char *buf, size_t buflen) -{ - if (!kn) - return strlcpy(buf, "(null)", buflen); - - return strlcpy(buf, kn->parent ? kn->name : "/", buflen); -} - /* kernfs_node_depth - compute depth from @from to @to */ static size_t kernfs_depth(struct kernfs_node *from, struct kernfs_node *to) { size_t depth = 0; - while (to->parent && to != from) { + while (rcu_dereference(to->__parent) && to != from) { depth++; - to = to->parent; + to = rcu_dereference(to->__parent); } return depth; } @@ -84,18 +78,18 @@ static struct kernfs_node *kernfs_common_ancestor(struct kernfs_node *a, db = kernfs_depth(rb->kn, b); while (da > db) { - a = a->parent; + a = rcu_dereference(a->__parent); da--; } while (db > da) { - b = b->parent; + b = rcu_dereference(b->__parent); db--; } /* worst case b and a will be the same at root */ while (b != a) { - b = b->parent; - a = a->parent; + b = rcu_dereference(b->__parent); + a = rcu_dereference(a->__parent); } return a; @@ -168,10 +162,13 @@ static int kernfs_path_from_node_locked(struct kernfs_node *kn_to, /* Calculate how many bytes we need for the rest */ for (i = depth_to - 1; i >= 0; i--) { + const char *name; + for (kn = kn_to, j = 0; j < i; j++) - kn = kn->parent; + kn = rcu_dereference(kn->__parent); - len += scnprintf(buf + len, buflen - len, "/%s", kn->name); + name = rcu_dereference(kn->name); + len += scnprintf(buf + len, buflen - len, "/%s", name); } return len; @@ -184,24 +181,29 @@ static int kernfs_path_from_node_locked(struct kernfs_node *kn_to, * @buflen: size of @buf * * Copies the name of @kn into @buf of @buflen bytes. The behavior is - * similar to strlcpy(). + * similar to strscpy(). * * Fills buffer with "(null)" if @kn is %NULL. * - * Return: the length of @kn's name and if @buf isn't long enough, - * it's filled up to @buflen-1 and nul terminated. + * Return: the resulting length of @buf. If @buf isn't long enough, + * it's filled up to @buflen-1 and nul terminated, and returns -E2BIG. * * This function can be called from any context. */ int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen) { - unsigned long flags; - int ret; + struct kernfs_node *kn_parent; - read_lock_irqsave(&kernfs_rename_lock, flags); - ret = kernfs_name_locked(kn, buf, buflen); - read_unlock_irqrestore(&kernfs_rename_lock, flags); - return ret; + if (!kn) + return strscpy(buf, "(null)", buflen); + + guard(rcu)(); + /* + * KERNFS_ROOT_INVARIANT_PARENT is ignored here. The name is RCU freed and + * the parent is either existing or not. + */ + kn_parent = rcu_dereference(kn->__parent); + return strscpy(buf, kn_parent ? rcu_dereference(kn->name) : "/", buflen); } /** @@ -223,13 +225,17 @@ int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen) int kernfs_path_from_node(struct kernfs_node *to, struct kernfs_node *from, char *buf, size_t buflen) { - unsigned long flags; - int ret; + struct kernfs_root *root; - read_lock_irqsave(&kernfs_rename_lock, flags); - ret = kernfs_path_from_node_locked(to, from, buf, buflen); - read_unlock_irqrestore(&kernfs_rename_lock, flags); - return ret; + guard(rcu)(); + if (to) { + root = kernfs_root(to); + if (!(root->flags & KERNFS_ROOT_INVARIANT_PARENT)) { + guard(read_lock_irqsave)(&kernfs_rename_lock); + return kernfs_path_from_node_locked(to, from, buf, buflen); + } + } + return kernfs_path_from_node_locked(to, from, buf, buflen); } EXPORT_SYMBOL_GPL(kernfs_path_from_node); @@ -295,7 +301,7 @@ struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn) unsigned long flags; read_lock_irqsave(&kernfs_rename_lock, flags); - parent = kn->parent; + parent = kernfs_parent(kn); kernfs_get(parent); read_unlock_irqrestore(&kernfs_rename_lock, flags); @@ -336,13 +342,13 @@ static int kernfs_name_compare(unsigned int hash, const char *name, return -1; if (ns > kn->ns) return 1; - return strcmp(name, kn->name); + return strcmp(name, kernfs_rcu_name(kn)); } static int kernfs_sd_compare(const struct kernfs_node *left, const struct kernfs_node *right) { - return kernfs_name_compare(left->hash, left->name, left->ns, right); + return kernfs_name_compare(left->hash, kernfs_rcu_name(left), left->ns, right); } /** @@ -360,8 +366,12 @@ static int kernfs_sd_compare(const struct kernfs_node *left, */ static int kernfs_link_sibling(struct kernfs_node *kn) { - struct rb_node **node = &kn->parent->dir.children.rb_node; struct rb_node *parent = NULL; + struct kernfs_node *kn_parent; + struct rb_node **node; + + kn_parent = kernfs_parent(kn); + node = &kn_parent->dir.children.rb_node; while (*node) { struct kernfs_node *pos; @@ -380,13 +390,13 @@ static int kernfs_link_sibling(struct kernfs_node *kn) /* add new node and rebalance the tree */ rb_link_node(&kn->rb, parent, node); - rb_insert_color(&kn->rb, &kn->parent->dir.children); + rb_insert_color(&kn->rb, &kn_parent->dir.children); /* successfully added, account subdir number */ down_write(&kernfs_root(kn)->kernfs_iattr_rwsem); if (kernfs_type(kn) == KERNFS_DIR) - kn->parent->dir.subdirs++; - kernfs_inc_rev(kn->parent); + kn_parent->dir.subdirs++; + kernfs_inc_rev(kn_parent); up_write(&kernfs_root(kn)->kernfs_iattr_rwsem); return 0; @@ -407,16 +417,19 @@ static int kernfs_link_sibling(struct kernfs_node *kn) */ static bool kernfs_unlink_sibling(struct kernfs_node *kn) { + struct kernfs_node *kn_parent; + if (RB_EMPTY_NODE(&kn->rb)) return false; + kn_parent = kernfs_parent(kn); down_write(&kernfs_root(kn)->kernfs_iattr_rwsem); if (kernfs_type(kn) == KERNFS_DIR) - kn->parent->dir.subdirs--; - kernfs_inc_rev(kn->parent); + kn_parent->dir.subdirs--; + kernfs_inc_rev(kn_parent); up_write(&kernfs_root(kn)->kernfs_iattr_rwsem); - rb_erase(&kn->rb, &kn->parent->dir.children); + rb_erase(&kn->rb, &kn_parent->dir.children); RB_CLEAR_NODE(&kn->rb); return true; } @@ -533,7 +546,8 @@ static void kernfs_free_rcu(struct rcu_head *rcu) { struct kernfs_node *kn = container_of(rcu, struct kernfs_node, rcu); - kfree_const(kn->name); + /* If the whole node goes away, then name can't be used outside */ + kfree_const(rcu_access_pointer(kn->name)); if (kn->iattr) { simple_xattrs_free(&kn->iattr->xattrs, NULL); @@ -562,11 +576,12 @@ void kernfs_put(struct kernfs_node *kn) * Moving/renaming is always done while holding reference. * kn->parent won't change beneath us. */ - parent = kn->parent; + parent = kernfs_parent(kn); WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS, "kernfs_put: %s/%s: released with incorrect active_ref %d\n", - parent ? parent->name : "", kn->name, atomic_read(&kn->active)); + parent ? rcu_dereference(parent->name) : "", + rcu_dereference(kn->name), atomic_read(&kn->active)); if (kernfs_type(kn) == KERNFS_LINK) kernfs_put(kn->symlink.target_kn); @@ -643,7 +658,7 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, atomic_set(&kn->active, KN_DEACTIVATED_BIAS); RB_CLEAR_NODE(&kn->rb); - kn->name = name; + rcu_assign_pointer(kn->name, name); kn->mode = mode; kn->flags = flags; @@ -701,7 +716,7 @@ struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, name, mode, uid, gid, flags); if (kn) { kernfs_get(parent); - kn->parent = parent; + rcu_assign_pointer(kn->__parent, parent); } return kn; } @@ -769,18 +784,20 @@ struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root, */ int kernfs_add_one(struct kernfs_node *kn) { - struct kernfs_node *parent = kn->parent; - struct kernfs_root *root = kernfs_root(parent); + struct kernfs_root *root = kernfs_root(kn); struct kernfs_iattrs *ps_iattr; + struct kernfs_node *parent; bool has_ns; int ret; down_write(&root->kernfs_rwsem); + parent = kernfs_parent(kn); ret = -EINVAL; has_ns = kernfs_ns_enabled(parent); if (WARN(has_ns != (bool)kn->ns, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n", - has_ns ? "required" : "invalid", parent->name, kn->name)) + has_ns ? "required" : "invalid", + kernfs_rcu_name(parent), kernfs_rcu_name(kn))) goto out_unlock; if (kernfs_type(parent) != KERNFS_DIR) @@ -790,7 +807,7 @@ int kernfs_add_one(struct kernfs_node *kn) if (parent->flags & (KERNFS_REMOVING | KERNFS_EMPTY_DIR)) goto out_unlock; - kn->hash = kernfs_name_hash(kn->name, kn->ns); + kn->hash = kernfs_name_hash(kernfs_rcu_name(kn), kn->ns); ret = kernfs_link_sibling(kn); if (ret) @@ -846,7 +863,7 @@ static struct kernfs_node *kernfs_find_ns(struct kernfs_node *parent, if (has_ns != (bool)ns) { WARN(1, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n", - has_ns ? "required" : "invalid", parent->name, name); + has_ns ? "required" : "invalid", kernfs_rcu_name(parent), name); return NULL; } @@ -871,16 +888,16 @@ static struct kernfs_node *kernfs_walk_ns(struct kernfs_node *parent, const unsigned char *path, const void *ns) { - size_t len; + ssize_t len; char *p, *name; lockdep_assert_held_read(&kernfs_root(parent)->kernfs_rwsem); spin_lock_irq(&kernfs_pr_cont_lock); - len = strlcpy(kernfs_pr_cont_buf, path, sizeof(kernfs_pr_cont_buf)); + len = strscpy(kernfs_pr_cont_buf, path, sizeof(kernfs_pr_cont_buf)); - if (len >= sizeof(kernfs_pr_cont_buf)) { + if (len < 0) { spin_unlock_irq(&kernfs_pr_cont_lock); return NULL; } @@ -949,6 +966,11 @@ struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent, return kn; } +unsigned int kernfs_root_flags(struct kernfs_node *kn) +{ + return kernfs_root(kn)->flags; +} + /** * kernfs_create_root - create a new kernfs hierarchy * @scops: optional syscall operations for the hierarchy @@ -1111,7 +1133,7 @@ struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent, static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags) { - struct kernfs_node *kn; + struct kernfs_node *kn, *parent; struct kernfs_root *root; if (flags & LOOKUP_RCU) @@ -1119,8 +1141,6 @@ static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags) /* Negative hashed dentry? */ if (d_really_is_negative(dentry)) { - struct kernfs_node *parent; - /* If the kernfs parent node has changed discard and * proceed to ->lookup. * @@ -1162,16 +1182,17 @@ static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags) if (!kernfs_active(kn)) goto out_bad; + parent = kernfs_parent(kn); /* The kernfs node has been moved? */ - if (kernfs_dentry_node(dentry->d_parent) != kn->parent) + if (kernfs_dentry_node(dentry->d_parent) != parent) goto out_bad; /* The kernfs node has been renamed */ - if (strcmp(dentry->d_name.name, kn->name) != 0) + if (strcmp(dentry->d_name.name, kernfs_rcu_name(kn)) != 0) goto out_bad; /* The kernfs node has been moved to a different namespace */ - if (kn->parent && kernfs_ns_enabled(kn->parent) && + if (parent && kernfs_ns_enabled(parent) && kernfs_info(dentry->d_sb)->ns != kn->ns) goto out_bad; @@ -1364,7 +1385,7 @@ static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos, return kernfs_leftmost_descendant(rb_to_kn(rbn)); /* no sibling left, visit parent */ - return pos->parent; + return kernfs_parent(pos); } static void kernfs_activate_one(struct kernfs_node *kn) @@ -1376,7 +1397,7 @@ static void kernfs_activate_one(struct kernfs_node *kn) if (kernfs_active(kn) || (kn->flags & (KERNFS_HIDDEN | KERNFS_REMOVING))) return; - WARN_ON_ONCE(kn->parent && RB_EMPTY_NODE(&kn->rb)); + WARN_ON_ONCE(rcu_access_pointer(kn->__parent) && RB_EMPTY_NODE(&kn->rb)); WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); atomic_sub(KN_DEACTIVATED_BIAS, &kn->active); @@ -1446,7 +1467,7 @@ void kernfs_show(struct kernfs_node *kn, bool show) static void __kernfs_remove(struct kernfs_node *kn) { - struct kernfs_node *pos; + struct kernfs_node *pos, *parent; /* Short-circuit if non-root @kn has already finished removal. */ if (!kn) @@ -1458,10 +1479,10 @@ static void __kernfs_remove(struct kernfs_node *kn) * This is for kernfs_remove_self() which plays with active ref * after removal. */ - if (kn->parent && RB_EMPTY_NODE(&kn->rb)) + if (kernfs_parent(kn) && RB_EMPTY_NODE(&kn->rb)) return; - pr_debug("kernfs %s: removing\n", kn->name); + pr_debug("kernfs %s: removing\n", kernfs_rcu_name(kn)); /* prevent new usage by marking all nodes removing and deactivating */ pos = NULL; @@ -1484,14 +1505,14 @@ static void __kernfs_remove(struct kernfs_node *kn) kernfs_get(pos); kernfs_drain(pos); - + parent = kernfs_parent(pos); /* * kernfs_unlink_sibling() succeeds once per node. Use it * to decide who's responsible for cleanups. */ - if (!pos->parent || kernfs_unlink_sibling(pos)) { + if (!parent || kernfs_unlink_sibling(pos)) { struct kernfs_iattrs *ps_iattr = - pos->parent ? pos->parent->iattr : NULL; + parent ? parent->iattr : NULL; /* update timestamps on the parent */ down_write(&kernfs_root(kn)->kernfs_iattr_rwsem); @@ -1718,11 +1739,11 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, { struct kernfs_node *old_parent; struct kernfs_root *root; - const char *old_name = NULL; + const char *old_name; int error; /* can't move or rename root */ - if (!kn->parent) + if (!rcu_access_pointer(kn->__parent)) return -EINVAL; root = kernfs_root(kn); @@ -1733,9 +1754,19 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, (new_parent->flags & KERNFS_EMPTY_DIR)) goto out; + old_parent = kernfs_parent(kn); + if (root->flags & KERNFS_ROOT_INVARIANT_PARENT) { + error = -EINVAL; + if (WARN_ON_ONCE(old_parent != new_parent)) + goto out; + } + error = 0; - if ((kn->parent == new_parent) && (kn->ns == new_ns) && - (strcmp(kn->name, new_name) == 0)) + old_name = kernfs_rcu_name(kn); + if (!new_name) + new_name = old_name; + if ((old_parent == new_parent) && (kn->ns == new_ns) && + (strcmp(old_name, new_name) == 0)) goto out; /* nothing to rename */ error = -EEXIST; @@ -1743,7 +1774,7 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, goto out; /* rename kernfs_node */ - if (strcmp(kn->name, new_name) != 0) { + if (strcmp(old_name, new_name) != 0) { error = -ENOMEM; new_name = kstrdup_const(new_name, GFP_KERNEL); if (!new_name) @@ -1756,27 +1787,32 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, * Move to the appropriate place in the appropriate directories rbtree. */ kernfs_unlink_sibling(kn); - kernfs_get(new_parent); - /* rename_lock protects ->parent and ->name accessors */ - write_lock_irq(&kernfs_rename_lock); + /* rename_lock protects ->parent accessors */ + if (old_parent != new_parent) { + kernfs_get(new_parent); + write_lock_irq(&kernfs_rename_lock); - old_parent = kn->parent; - kn->parent = new_parent; + rcu_assign_pointer(kn->__parent, new_parent); - kn->ns = new_ns; - if (new_name) { - old_name = kn->name; - kn->name = new_name; - } + kn->ns = new_ns; + if (new_name) + rcu_assign_pointer(kn->name, new_name); - write_unlock_irq(&kernfs_rename_lock); + write_unlock_irq(&kernfs_rename_lock); + kernfs_put(old_parent); + } else { + /* name assignment is RCU protected, parent is the same */ + kn->ns = new_ns; + if (new_name) + rcu_assign_pointer(kn->name, new_name); + } - kn->hash = kernfs_name_hash(kn->name, kn->ns); + kn->hash = kernfs_name_hash(new_name ?: old_name, kn->ns); kernfs_link_sibling(kn); - kernfs_put(old_parent); - kfree_const(old_name); + if (new_name && !is_kernel_rodata((unsigned long)old_name)) + kfree_rcu_mightsleep(old_name); error = 0; out: @@ -1795,7 +1831,8 @@ static struct kernfs_node *kernfs_dir_pos(const void *ns, { if (pos) { int valid = kernfs_active(pos) && - pos->parent == parent && hash == pos->hash; + rcu_access_pointer(pos->__parent) == parent && + hash == pos->hash; kernfs_put(pos); if (!valid) pos = NULL; @@ -1860,7 +1897,7 @@ static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx) for (pos = kernfs_dir_pos(ns, parent, ctx->pos, pos); pos; pos = kernfs_dir_next_pos(ns, parent, ctx->pos, pos)) { - const char *name = pos->name; + const char *name = kernfs_rcu_name(pos); unsigned int type = fs_umode_to_dtype(pos->mode); int len = strlen(name); ino_t ino = kernfs_ino(pos); @@ -1869,10 +1906,10 @@ static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx) file->private_data = pos; kernfs_get(pos); - up_read(&root->kernfs_rwsem); - if (!dir_emit(ctx, name, len, ino, type)) + if (!dir_emit(ctx, name, len, ino, type)) { + up_read(&root->kernfs_rwsem); return 0; - down_read(&root->kernfs_rwsem); + } } up_read(&root->kernfs_rwsem); file->private_data = NULL; diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index aa4b9da6743754c0df112ca56a14bdf1e601a495..29e6a931539822e69bfda1fdd107e3c77a1d51b4 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -636,11 +636,18 @@ static int kernfs_fop_open(struct inode *inode, struct file *file) * each file a separate locking class. Let's differentiate on * whether the file has mmap or not for now. * - * Both paths of the branch look the same. They're supposed to + * For similar reasons, writable and readonly files are given different + * lockdep key, because the writable file /sys/power/resume may call vfs + * lookup helpers for arbitrary paths and readonly files can be read by + * overlayfs from vfs helpers when sysfs is a lower layer of overalyfs. + * + * All three cases look the same. They're supposed to * look that way and give @of->mutex different static lockdep keys. */ if (has_mmap) mutex_init(&of->mutex); + else if (file->f_mode & FMODE_WRITE) + mutex_init(&of->mutex); else mutex_init(&of->mutex); @@ -857,6 +864,33 @@ static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait) return ret; } +static loff_t kernfs_fop_llseek(struct file *file, loff_t offset, int whence) +{ + struct kernfs_open_file *of = kernfs_of(file); + const struct kernfs_ops *ops; + loff_t ret; + + /* + * @of->mutex nests outside active ref and is primarily to ensure that + * the ops aren't called concurrently for the same open file. + */ + mutex_lock(&of->mutex); + if (!kernfs_get_active(of->kn)) { + mutex_unlock(&of->mutex); + return -ENODEV; + } + + ops = kernfs_ops(of->kn); + if (ops->llseek) + ret = ops->llseek(of, offset, whence); + else + ret = generic_file_llseek(file, offset, whence); + + kernfs_put_active(of->kn); + mutex_unlock(&of->mutex); + return ret; +} + static void kernfs_notify_workfn(struct work_struct *work) { struct kernfs_node *kn; @@ -878,9 +912,11 @@ static void kernfs_notify_workfn(struct work_struct *work) /* kick fsnotify */ down_read(&root->kernfs_supers_rwsem); + down_read(&root->kernfs_rwsem); list_for_each_entry(info, &kernfs_root(kn)->supers, node) { struct kernfs_node *parent; struct inode *p_inode = NULL; + const char *kn_name; struct inode *inode; struct qstr name; @@ -894,7 +930,8 @@ static void kernfs_notify_workfn(struct work_struct *work) if (!inode) continue; - name = (struct qstr)QSTR_INIT(kn->name, strlen(kn->name)); + kn_name = kernfs_rcu_name(kn); + name = (struct qstr)QSTR_INIT(kn_name, strlen(kn_name)); parent = kernfs_get_parent(kn); if (parent) { p_inode = ilookup(info->sb, kernfs_ino(parent)); @@ -914,6 +951,7 @@ static void kernfs_notify_workfn(struct work_struct *work) iput(inode); } + up_read(&root->kernfs_rwsem); up_read(&root->kernfs_supers_rwsem); kernfs_put(kn); goto repeat; @@ -959,7 +997,7 @@ EXPORT_SYMBOL_GPL(kernfs_notify); const struct file_operations kernfs_file_fops = { .read_iter = kernfs_fop_read_iter, .write_iter = kernfs_fop_write_iter, - .llseek = generic_file_llseek, + .llseek = kernfs_fop_llseek, .mmap = kernfs_fop_mmap, .open = kernfs_fop_open, .release = kernfs_fop_release, diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index 210dac7e9ee25c9ba3ee2f0af1d68a60e3e1f9c3..3de6346e41e5ed91564454eb6b76a58fda4eb6c1 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -19,6 +19,8 @@ #include #include +extern rwlock_t kernfs_rename_lock; + struct kernfs_iattrs { kuid_t ia_uid; kgid_t ia_gid; @@ -64,11 +66,14 @@ struct kernfs_root { * * Return: the kernfs_root @kn belongs to. */ -static inline struct kernfs_root *kernfs_root(struct kernfs_node *kn) +static inline struct kernfs_root *kernfs_root(const struct kernfs_node *kn) { + const struct kernfs_node *knp; /* if parent exists, it's always a dir; otherwise, @sd is a dir */ - if (kn->parent) - kn = kn->parent; + guard(rcu)(); + knp = rcu_dereference(kn->__parent); + if (knp) + kn = knp; return kn->dir.root; } @@ -97,6 +102,32 @@ struct kernfs_super_info { }; #define kernfs_info(SB) ((struct kernfs_super_info *)(SB->s_fs_info)) +static inline bool kernfs_root_is_locked(const struct kernfs_node *kn) +{ + return lockdep_is_held(&kernfs_root(kn)->kernfs_rwsem); +} + +static inline const char *kernfs_rcu_name(const struct kernfs_node *kn) +{ + return rcu_dereference_check(kn->name, kernfs_root_is_locked(kn)); +} + +static inline struct kernfs_node *kernfs_parent(const struct kernfs_node *kn) +{ + /* + * The kernfs_node::__parent remains valid within a RCU section. The kn + * can be reparented (and renamed) which changes the entry. This can be + * avoided by locking kernfs_root::kernfs_rwsem or kernfs_rename_lock. + * Both locks can be used to obtain a reference on __parent. Once the + * reference count reaches 0 then the node is about to be freed + * and can not be renamed (or become a different parent) anymore. + */ + return rcu_dereference_check(kn->__parent, + kernfs_root_is_locked(kn) || + lockdep_is_held(&kernfs_rename_lock) || + !atomic_read(&kn->count)); +} + static inline struct kernfs_node *kernfs_dentry_node(struct dentry *dentry) { if (d_really_is_negative(dentry)) diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index 9aa3384856b73e4de3e9911c6ac238a8b9159df9..811a1632d97a9ec8a4bcc74309128b79acc1b16c 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -147,8 +147,10 @@ static struct dentry *kernfs_fh_to_parent(struct super_block *sb, static struct dentry *kernfs_get_parent_dentry(struct dentry *child) { struct kernfs_node *kn = kernfs_dentry_node(child); + struct kernfs_root *root = kernfs_root(kn); - return d_obtain_alias(kernfs_get_inode(child->d_sb, kn->parent)); + guard(rwsem_read)(&root->kernfs_rwsem); + return d_obtain_alias(kernfs_get_inode(child->d_sb, kernfs_parent(kn))); } static const struct export_operations kernfs_export_ops = { @@ -188,10 +190,10 @@ static struct kernfs_node *find_next_ancestor(struct kernfs_node *child, return NULL; } - while (child->parent != parent) { - if (!child->parent) + while (kernfs_parent(child) != parent) { + child = kernfs_parent(child); + if (!child) return NULL; - child = child->parent; } return child; @@ -208,17 +210,28 @@ struct dentry *kernfs_node_dentry(struct kernfs_node *kn, struct super_block *sb) { struct dentry *dentry; - struct kernfs_node *knparent = NULL; + struct kernfs_node *knparent; + struct kernfs_root *root; BUG_ON(sb->s_op != &kernfs_sops); dentry = dget(sb->s_root); /* Check if this is the root kernfs_node */ - if (!kn->parent) + if (!rcu_access_pointer(kn->__parent)) return dentry; - knparent = find_next_ancestor(kn, NULL); + root = kernfs_root(kn); + /* + * As long as kn is valid, its parent can not vanish. This is cgroup's + * kn so it can't have its parent replaced. Therefore it is safe to use + * the ancestor node outside of the RCU or locked section. + */ + if (WARN_ON_ONCE(!(root->flags & KERNFS_ROOT_INVARIANT_PARENT))) + return ERR_PTR(-EINVAL); + scoped_guard(rcu) { + knparent = find_next_ancestor(kn, NULL); + } if (WARN_ON(!knparent)) { dput(dentry); return ERR_PTR(-EINVAL); @@ -227,17 +240,26 @@ struct dentry *kernfs_node_dentry(struct kernfs_node *kn, do { struct dentry *dtmp; struct kernfs_node *kntmp; + const char *name; if (kn == knparent) return dentry; - kntmp = find_next_ancestor(kn, knparent); - if (WARN_ON(!kntmp)) { + + scoped_guard(rwsem_read, &root->kernfs_rwsem) { + kntmp = find_next_ancestor(kn, knparent); + if (WARN_ON(!kntmp)) { + dput(dentry); + return ERR_PTR(-EINVAL); + } + name = kstrdup(kernfs_rcu_name(kntmp), GFP_KERNEL); + } + if (!name) { dput(dentry); - return ERR_PTR(-EINVAL); + return ERR_PTR(-ENOMEM); } - dtmp = lookup_positive_unlocked(kntmp->name, dentry, - strlen(kntmp->name)); + dtmp = lookup_positive_unlocked(name, dentry, strlen(name)); dput(dentry); + kfree(name); if (IS_ERR(dtmp)) return dtmp; knparent = kntmp; diff --git a/fs/kernfs/symlink.c b/fs/kernfs/symlink.c index 45371a70caa710e7c6633d631759b8f97c88d68f..0bd8a2143723d178dedbd9e5554835e2faa17e91 100644 --- a/fs/kernfs/symlink.c +++ b/fs/kernfs/symlink.c @@ -62,10 +62,10 @@ static int kernfs_get_target_path(struct kernfs_node *parent, /* go up to the root, stop at the base */ base = parent; - while (base->parent) { - kn = target->parent; - while (kn->parent && base != kn) - kn = kn->parent; + while (kernfs_parent(base)) { + kn = kernfs_parent(target); + while (kernfs_parent(kn) && base != kn) + kn = kernfs_parent(kn); if (base == kn) break; @@ -75,14 +75,14 @@ static int kernfs_get_target_path(struct kernfs_node *parent, strcpy(s, "../"); s += 3; - base = base->parent; + base = kernfs_parent(base); } /* determine end of target string for reverse fillup */ kn = target; - while (kn->parent && kn != base) { - len += strlen(kn->name) + 1; - kn = kn->parent; + while (kernfs_parent(kn) && kn != base) { + len += strlen(kernfs_rcu_name(kn)) + 1; + kn = kernfs_parent(kn); } /* check limits */ @@ -94,15 +94,16 @@ static int kernfs_get_target_path(struct kernfs_node *parent, /* reverse fillup of target string from target to base */ kn = target; - while (kn->parent && kn != base) { - int slen = strlen(kn->name); + while (kernfs_parent(kn) && kn != base) { + const char *name = kernfs_rcu_name(kn); + int slen = strlen(name); len -= slen; - memcpy(s + len, kn->name, slen); + memcpy(s + len, name, slen); if (len) s[--len] = '/'; - kn = kn->parent; + kn = kernfs_parent(kn); } return 0; @@ -111,12 +112,13 @@ static int kernfs_get_target_path(struct kernfs_node *parent, static int kernfs_getlink(struct inode *inode, char *path) { struct kernfs_node *kn = inode->i_private; - struct kernfs_node *parent = kn->parent; + struct kernfs_node *parent; struct kernfs_node *target = kn->symlink.target_kn; - struct kernfs_root *root = kernfs_root(parent); + struct kernfs_root *root = kernfs_root(kn); int error; down_read(&root->kernfs_rwsem); + parent = kernfs_parent(kn); error = kernfs_get_target_path(parent, target, path); up_read(&root->kernfs_rwsem); diff --git a/fs/resctrl/Kconfig b/fs/resctrl/Kconfig index 36a1ddbe6c216faf4ca69eeff8273b2400481491..21671301bd8a490a397981ddd7a95b2ccdbce375 100644 --- a/fs/resctrl/Kconfig +++ b/fs/resctrl/Kconfig @@ -2,22 +2,38 @@ config RESCTRL_FS bool "CPU Resource Control Filesystem (resctrl)" depends on ARCH_HAS_CPU_RESCTRL select KERNFS - select PROC_CPU_RESCTRL if PROC_FS + select PROC_CPU_RESCTRL if PROC_FS help - Resctrl is a filesystem interface - to control allocation and - monitoring of system resources - used by the CPUs. + Some architectures provide hardware facilities to group tasks and + monitor and control their usage of memory system resources such as + caches and memory bandwidth. Examples of such facilities include + Intel's Resource Director Technology (Intel(R) RDT) and AMD's + Platform Quality of Service (AMD QoS). + + If your system has the necessary support and you want to be able to + assign tasks to groups and manipulate the associated resource + monitors and controls from userspace, say Y here to get a mountable + 'resctrl' filesystem that lets you do just that. + + If nothing mounts or prods the 'resctrl' filesystem, resource + controls and monitors are left in a quiescent, permissive state. + + On architectures where this can be disabled independently, it is + safe to say N. + + See for more information. config RESCTRL_FS_PSEUDO_LOCK bool + depends on RESCTRL_FS help - Software mechanism to pin data in a cache portion using - micro-architecture specific knowledge. + Software mechanism to pin data in a cache portion using + micro-architecture specific knowledge. config RESCTRL_RMID_DEPENDS_ON_CLOSID bool + depends on RESCTRL_FS help - Enable by the architecture when the RMID values depend on the CLOSID. - This causes the closid allocator to search for CLOSID with clean + Enabled by the architecture when the RMID values depend on the CLOSID. + This causes the CLOSID allocator to search for CLOSID with clean RMID. diff --git a/fs/resctrl/Makefile b/fs/resctrl/Makefile index 10fcfb0fdb102a4329e2cbbfdc119e3b288d7db2..e67f34d2236a2044bb4925aa781e75a92a7e5dcd 100644 --- a/fs/resctrl/Makefile +++ b/fs/resctrl/Makefile @@ -1,3 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_RESCTRL_FS) += rdtgroup.o ctrlmondata.o monitor.o -obj-$(CONFIG_RESCTRL_FS_PSEUDO_LOCK) += psuedo_lock.o +obj-$(CONFIG_RESCTRL_FS_PSEUDO_LOCK) += pseudo_lock.o + +# To allow define_trace.h's recursive include: +CFLAGS_monitor.o = -I$(src) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 6651e39c1cb8e24243a043653c1c59f192807634..3c39cfacb2518338e40593ac32245fec62d06aa6 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -19,16 +19,18 @@ #include #include #include +#include + #include "internal.h" struct rdt_parse_data { - u32 closid; - enum rdtgrp_mode mode; - char *buf; + struct rdtgroup *rdtgrp; + char *buf; }; -typedef int(ctrlval_parser_t)(struct rdt_parse_data *data, - struct resctrl_schema *s, struct rdt_domain *d); +typedef int (ctrlval_parser_t)(struct rdt_parse_data *data, + struct resctrl_schema *s, + struct rdt_ctrl_domain *d); /* * Check whether MBA bandwidth percentage value is correct. The value is @@ -61,9 +63,9 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) return true; } - if (bw < r->membw.min_bw || bw > r->default_ctrl) { - rdt_last_cmd_printf("MB value %u out of range [%d,%d]\n", bw, - r->membw.min_bw, r->default_ctrl); + if (bw < r->membw.min_bw || bw > r->membw.max_bw) { + rdt_last_cmd_printf("MB value %u out of range [%d,%d]\n", + bw, r->membw.min_bw, r->membw.max_bw); return false; } @@ -72,16 +74,16 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) } static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, - struct rdt_domain *d) + struct rdt_ctrl_domain *d) { struct resctrl_staged_config *cfg; + u32 closid = data->rdtgrp->closid; struct rdt_resource *r = s->res; - u32 closid = data->closid; u32 bw_val; cfg = &d->staged_config[s->conf_type]; if (cfg->have_new_ctrl) { - rdt_last_cmd_printf("Duplicate domain %d\n", d->id); + rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); return -EINVAL; } @@ -111,8 +113,9 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, */ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r) { - unsigned long first_bit, zero_bit, val; + u32 supported_bits = BIT_MASK(r->cache.cbm_len) - 1; unsigned int cbm_len = r->cache.cbm_len; + unsigned long first_bit, zero_bit, val; int ret; ret = kstrtoul(buf, 16, &val); @@ -121,7 +124,7 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r) return false; } - if ((r->cache.min_cbm_bits > 0 && val == 0) || val > r->default_ctrl) { + if ((r->cache.min_cbm_bits > 0 && val == 0) || val > supported_bits) { rdt_last_cmd_puts("Mask out of range\n"); return false; } @@ -132,8 +135,7 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r) /* Are non-contiguous bitmasks allowed? */ if (!r->cache.arch_has_sparse_bitmasks && (find_next_bit(&val, cbm_len, zero_bit) < cbm_len)) { - rdt_last_cmd_printf("The mask %lx has non-consecutive 1-bits\n", - val); + rdt_last_cmd_printf("The mask %lx has non-consecutive 1-bits\n", val); return false; } @@ -152,17 +154,16 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r) * resource type. */ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, - struct rdt_domain *d) + struct rdt_ctrl_domain *d) { - enum rdtgrp_mode mode = data->mode; + struct rdtgroup *rdtgrp = data->rdtgrp; struct resctrl_staged_config *cfg; struct rdt_resource *r = s->res; - u32 closid = data->closid; u32 cbm_val; cfg = &d->staged_config[s->conf_type]; if (cfg->have_new_ctrl) { - rdt_last_cmd_printf("Duplicate domain %d\n", d->id); + rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); return -EINVAL; } @@ -170,7 +171,7 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, * Cannot set up more than one pseudo-locked region in a cache * hierarchy. */ - if (mode == RDT_MODE_PSEUDO_LOCKSETUP && + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP && rdtgroup_pseudo_locked_in_hierarchy(d)) { rdt_last_cmd_puts("Pseudo-locked region in hierarchy\n"); return -EINVAL; @@ -179,7 +180,8 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, if (!cbm_validate(data->buf, &cbm_val, r)) return -EINVAL; - if ((mode == RDT_MODE_EXCLUSIVE || mode == RDT_MODE_SHAREABLE) && + if ((rdtgrp->mode == RDT_MODE_EXCLUSIVE || + rdtgrp->mode == RDT_MODE_SHAREABLE) && rdtgroup_cbm_overlaps_pseudo_locked(d, cbm_val)) { rdt_last_cmd_puts("CBM overlaps with pseudo-locked region\n"); return -EINVAL; @@ -189,14 +191,14 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, * The CBM may not overlap with the CBM of another closid if * either is exclusive. */ - if (rdtgroup_cbm_overlaps(s, d, cbm_val, closid, true)) { + if (rdtgroup_cbm_overlaps(s, d, cbm_val, rdtgrp->closid, true)) { rdt_last_cmd_puts("Overlaps with exclusive group\n"); return -EINVAL; } - if (rdtgroup_cbm_overlaps(s, d, cbm_val, closid, false)) { - if (mode == RDT_MODE_EXCLUSIVE || - mode == RDT_MODE_PSEUDO_LOCKSETUP) { + if (rdtgroup_cbm_overlaps(s, d, cbm_val, rdtgrp->closid, false)) { + if (rdtgrp->mode == RDT_MODE_EXCLUSIVE || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { rdt_last_cmd_puts("Overlaps with other group\n"); return -EINVAL; } @@ -208,14 +210,6 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, return 0; } -static ctrlval_parser_t *get_parser(struct rdt_resource *res) -{ - if (res->fflags & RFTYPE_RES_CACHE) - return &parse_cbm; - else - return &parse_bw; -} - /* * For each domain in this resource we expect to find a series of: * id=mask @@ -225,18 +219,30 @@ static ctrlval_parser_t *get_parser(struct rdt_resource *res) static int parse_line(char *line, struct resctrl_schema *s, struct rdtgroup *rdtgrp) { - ctrlval_parser_t *parse_ctrlval = get_parser(s->res); enum resctrl_conf_type t = s->conf_type; + ctrlval_parser_t *parse_ctrlval = NULL; struct resctrl_staged_config *cfg; struct rdt_resource *r = s->res; struct rdt_parse_data data; + struct rdt_ctrl_domain *d; char *dom = NULL, *id; - struct rdt_domain *d; unsigned long dom_id; /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); + switch (r->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + parse_ctrlval = &parse_cbm; + break; + case RESCTRL_SCHEMA_RANGE: + parse_ctrlval = &parse_bw; + break; + } + + if (WARN_ON_ONCE(!parse_ctrlval)) + return -EINVAL; + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP && (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA)) { rdt_last_cmd_puts("Cannot pseudo-lock MBA resource\n"); @@ -253,14 +259,13 @@ static int parse_line(char *line, struct resctrl_schema *s, return -EINVAL; } dom = strim(dom); - list_for_each_entry(d, &r->domains, list) { - if (d->id == dom_id) { + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + if (d->hdr.id == dom_id) { data.buf = dom; - data.closid = rdtgrp->closid; - data.mode = rdtgrp->mode; + data.rdtgrp = rdtgrp; if (parse_ctrlval(&data, s, d)) return -EINVAL; - if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { cfg = &d->staged_config[t]; /* * In pseudo-locking setup mode and just @@ -376,20 +381,18 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, return ret ?: nbytes; } -static void show_doms(struct seq_file *s, struct resctrl_schema *schema, - char *resource_name, int closid) +static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int closid) { struct rdt_resource *r = schema->res; - struct rdt_domain *dom; + struct rdt_ctrl_domain *dom; bool sep = false; u32 ctrl_val; /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - if (resource_name) - seq_printf(s, "%*s:", max_name_width, resource_name); - list_for_each_entry(dom, &r->domains, list) { + seq_printf(s, "%*s:", max_name_width, schema->name); + list_for_each_entry(dom, &r->ctrl_domains, hdr.list) { if (sep) seq_puts(s, ";"); @@ -399,8 +402,7 @@ static void show_doms(struct seq_file *s, struct resctrl_schema *schema, ctrl_val = resctrl_arch_get_config(r, dom, closid, schema->conf_type); - seq_printf(s, r->format_str, dom->id, max_data_width, - ctrl_val); + seq_printf(s, schema->fmt_str, dom->hdr.id, ctrl_val); sep = true; } seq_puts(s, "\n"); @@ -428,14 +430,14 @@ int rdtgroup_schemata_show(struct kernfs_open_file *of, } else { seq_printf(s, "%s:%d=%x\n", rdtgrp->plr->s->res->name, - rdtgrp->plr->d->id, + rdtgrp->plr->d->hdr.id, rdtgrp->plr->cbm); } } else { closid = rdtgrp->closid; list_for_each_entry(schema, &resctrl_schema_all, list) { if (closid < schema->num_closid) - show_doms(s, schema, schema->name, closid); + show_doms(s, schema, closid); } } } else { @@ -452,9 +454,101 @@ static int smp_mon_event_count(void *arg) return 0; } +ssize_t rdtgroup_mba_mbps_event_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') + return -EINVAL; + buf[nbytes - 1] = '\0'; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + rdt_last_cmd_clear(); + + if (!strcmp(buf, "mbm_local_bytes")) { + if (resctrl_arch_is_mbm_local_enabled()) + rdtgrp->mba_mbps_event = QOS_L3_MBM_LOCAL_EVENT_ID; + else + ret = -EINVAL; + } else if (!strcmp(buf, "mbm_total_bytes")) { + if (resctrl_arch_is_mbm_total_enabled()) + rdtgrp->mba_mbps_event = QOS_L3_MBM_TOTAL_EVENT_ID; + else + ret = -EINVAL; + } else { + ret = -EINVAL; + } + + if (ret) + rdt_last_cmd_printf("Unsupported event id '%s'\n", buf); + + rdtgroup_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +int rdtgroup_mba_mbps_event_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + + if (rdtgrp) { + switch (rdtgrp->mba_mbps_event) { + case QOS_L3_MBM_LOCAL_EVENT_ID: + seq_puts(s, "mbm_local_bytes\n"); + break; + case QOS_L3_MBM_TOTAL_EVENT_ID: + seq_puts(s, "mbm_total_bytes\n"); + break; + default: + pr_warn_once("Bad event %d\n", rdtgrp->mba_mbps_event); + ret = -EINVAL; + break; + } + } else { + ret = -ENOENT; + } + + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id, + struct list_head **pos) +{ + struct rdt_domain_hdr *d; + struct list_head *l; + + list_for_each(l, h) { + d = list_entry(l, struct rdt_domain_hdr, list); + /* When id is found, return its domain. */ + if (id == d->id) + return d; + /* Stop searching when finding id's position in sorted list. */ + if (id < d->id) + break; + } + + if (pos) + *pos = l; + + return NULL; +} + void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, - struct rdt_domain *d, struct rdtgroup *rdtgrp, - int evtid, int first) + struct rdt_mon_domain *d, struct rdtgroup *rdtgrp, + cpumask_t *cpumask, int evtid, int first) { int cpu; @@ -468,7 +562,6 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, rr->evtid = evtid; rr->r = r; rr->d = d; - rr->val = 0; rr->first = first; rr->arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, evtid); if (IS_ERR(rr->arch_mon_ctx)) { @@ -476,7 +569,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, return; } - cpu = cpumask_any_housekeeping(&d->cpu_mask, RESCTRL_PICK_ANY_CPU); + cpu = cpumask_any_housekeeping(cpumask, RESCTRL_PICK_ANY_CPU); /* * cpumask_any_housekeeping() prefers housekeeping CPUs, but @@ -485,7 +578,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, * counters on some platforms if its called in IRQ context. */ if (tick_nohz_full_cpu(cpu)) - smp_call_function_any(&d->cpu_mask, mon_event_count, rr, 1); + smp_call_function_any(cpumask, mon_event_count, rr, 1); else smp_call_on_cpu(cpu, smp_mon_event_count, rr, false); @@ -495,13 +588,16 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, int rdtgroup_mondata_show(struct seq_file *m, void *arg) { struct kernfs_open_file *of = m->private; - u32 resid, evtid, domid; + enum resctrl_res_level resid; + enum resctrl_event_id evtid; + struct rdt_domain_hdr *hdr; + struct rmid_read rr = {0}; + struct rdt_mon_domain *d; struct rdtgroup *rdtgrp; + int domid, cpu, ret = 0; struct rdt_resource *r; - union mon_data_bits md; - struct rdt_domain *d; - struct rmid_read rr; - int ret = 0, index; + struct cacheinfo *ci; + struct mon_data *md; rdtgrp = rdtgroup_kn_lock_live(of->kn); if (!rdtgrp) { @@ -509,36 +605,58 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) goto out; } - md.priv = of->kn->priv; - resid = md.u.rid; - domid = md.u.domid; - evtid = md.u.evtid; + md = of->kn->priv; + if (WARN_ON_ONCE(!md)) { + ret = -EIO; + goto out; + } + resid = md->rid; + domid = md->domid; + evtid = md->evtid; r = resctrl_arch_get_resource(resid); - d = resctrl_arch_find_domain(r, domid); - if (IS_ERR_OR_NULL(d)) { + + if (md->sum) { + /* + * This file requires summing across all domains that share + * the L3 cache id that was provided in the "domid" field of the + * struct mon_data. Search all domains in the resource for + * one that matches this cache id. + */ + list_for_each_entry(d, &r->mon_domains, hdr.list) { + if (d->ci_id == domid) { + cpu = cpumask_any(&d->hdr.cpu_mask); + ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE); + if (!ci) + continue; + rr.ci = ci; + mon_event_read(&rr, r, NULL, rdtgrp, + &ci->shared_cpu_map, evtid, false); + goto checkresult; + } + } ret = -ENOENT; goto out; - } - - if (resctrl_arch_get_abmc_enabled() && evtid != QOS_L3_OCCUP_EVENT_ID) { - index = mon_event_config_index_get(evtid); - if (index != INVALID_CONFIG_INDEX && - rdtgrp->mon.cntr_id[index] == MON_CNTR_UNSET) { - rr.err = -ENOENT; - goto checkresult; + } else { + /* + * This file provides data from a single domain. Search + * the resource to find the domain with "domid". + */ + hdr = resctrl_find_domain(&r->mon_domains, domid, NULL); + if (!hdr || WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN)) { + ret = -ENOENT; + goto out; } + d = container_of(hdr, struct rdt_mon_domain, hdr); + mon_event_read(&rr, r, d, rdtgrp, &d->hdr.cpu_mask, evtid, false); } - mon_event_read(&rr, r, d, rdtgrp, evtid, false); - checkresult: + if (rr.err == -EIO) seq_puts(m, "Error\n"); else if (rr.err == -EINVAL) seq_puts(m, "Unavailable\n"); - else if (rr.err == -ENOENT) - seq_puts(m, "Unassigned\n"); else seq_printf(m, "%llu\n", rr.val); @@ -546,282 +664,3 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) rdtgroup_kn_unlock(of->kn); return ret; } - -#ifdef CONFIG_X86 -int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) -{ - struct resctrl_schema *s = of->kn->parent->priv; - struct rdt_resource *r = s->res; - - mutex_lock(&rdtgroup_mutex); - - if (r->cache.io_alloc_capable) { - if (resctrl_arch_get_io_alloc_enabled(r)) - seq_puts(seq, "enabled\n"); - else - seq_puts(seq, "disabled\n"); - } else { - seq_puts(seq, "not supported\n"); - } - - mutex_unlock(&rdtgroup_mutex); - - return 0; -} - -/* - * resctrl_io_alloc_closid_supported() - io_alloc feature utilizes the - * highest CLOSID value to direct I/O traffic. Ensure that io_alloc_closid - * is in the supported range. - */ -static bool resctrl_io_alloc_closid_supported(u32 io_alloc_closid) -{ - return io_alloc_closid < closids_supported(); -} - -/* - * Initialize io_alloc CLOSID cache resource CBM with all usable (shared - * and unused) cache portions. - */ -static int resctrl_io_alloc_init_cbm(struct resctrl_schema *s, u32 closid) -{ - enum resctrl_conf_type peer_type; - struct rdt_resource *r = s->res; - struct rdt_domain *d; - int ret; - - rdt_staged_configs_clear(); - - ret = rdtgroup_init_cat(s, closid); - if (ret < 0) - goto out; - - /* Keep CDP_CODE and CDP_DATA of io_alloc CLOSID's CBM in sync. */ - if (resctrl_arch_get_cdp_enabled(r->rid)) { - peer_type = resctrl_peer_type(s->conf_type); - list_for_each_entry(d, &s->res->domains, list) - memcpy(&d->staged_config[peer_type], - &d->staged_config[s->conf_type], - sizeof(d->staged_config[0])); - } - - ret = resctrl_arch_update_domains(r, closid); -out: - rdt_staged_configs_clear(); - return ret; -} - -/* - * resctrl_io_alloc_closid() - io_alloc feature routes I/O traffic using - * the highest available CLOSID. Retrieve the maximum CLOSID supported by the - * resource. Note that if Code Data Prioritization (CDP) is enabled, the number - * of available CLOSIDs is reduced by half. - */ -static u32 resctrl_io_alloc_closid(struct rdt_resource *r) -{ - if (resctrl_arch_get_cdp_enabled(r->rid)) - return resctrl_arch_get_num_closid(r) / 2 - 1; - else - return resctrl_arch_get_num_closid(r) - 1; -} - -ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, loff_t off) -{ - struct resctrl_schema *s = of->kn->parent->priv; - struct rdt_resource *r = s->res; - char const *grp_name; - u32 io_alloc_closid; - bool enable; - int ret; - - ret = kstrtobool(buf, &enable); - if (ret) - return ret; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - - rdt_last_cmd_clear(); - - if (!r->cache.io_alloc_capable) { - rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); - ret = -ENODEV; - goto out_unlock; - } - - /* If the feature is already up to date, no action is needed. */ - if (resctrl_arch_get_io_alloc_enabled(r) == enable) - goto out_unlock; - - io_alloc_closid = resctrl_io_alloc_closid(r); - if (!resctrl_io_alloc_closid_supported(io_alloc_closid)) { - rdt_last_cmd_printf("io_alloc CLOSID (ctrl_hw_id) %u is not available\n", - io_alloc_closid); - ret = -EINVAL; - goto out_unlock; - } - - if (enable) { - if (!closid_alloc_fixed(io_alloc_closid)) { - grp_name = rdtgroup_name_by_closid(io_alloc_closid); - WARN_ON_ONCE(!grp_name); - rdt_last_cmd_printf("CLOSID (ctrl_hw_id) %u for io_alloc is used by %s group\n", - io_alloc_closid, grp_name ? grp_name : "another"); - ret = -ENOSPC; - goto out_unlock; - } - - ret = resctrl_io_alloc_init_cbm(s, io_alloc_closid); - if (ret) { - rdt_last_cmd_puts("Failed to initialize io_alloc allocations\n"); - closid_free(io_alloc_closid); - goto out_unlock; - } - } else { - closid_free(io_alloc_closid); - } - - ret = resctrl_arch_io_alloc_enable(r, enable); - if (enable && ret) { - rdt_last_cmd_puts("Failed to enable io_alloc feature\n"); - closid_free(io_alloc_closid); - } - -out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); - - return ret ?: nbytes; -} - -int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) -{ - struct resctrl_schema *s = of->kn->parent->priv; - struct rdt_resource *r = s->res; - int ret = 0; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - - rdt_last_cmd_clear(); - - if (!r->cache.io_alloc_capable) { - rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); - ret = -ENODEV; - goto out_unlock; - } - - if (!resctrl_arch_get_io_alloc_enabled(r)) { - rdt_last_cmd_printf("io_alloc is not enabled on %s\n", s->name); - ret = -EINVAL; - goto out_unlock; - } - - /* - * When CDP is enabled, the CBMs of the highest CLOSID of CDP_CODE and - * CDP_DATA are kept in sync. As a result, the io_alloc CBMs shown for - * either CDP resource are identical and accurately represent the CBMs - * used for I/O. - */ - show_doms(seq, s, NULL, resctrl_io_alloc_closid(r)); - -out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); - return ret; -} - -static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, - struct resctrl_schema *s, u32 closid) -{ - enum resctrl_conf_type peer_type; - struct rdt_parse_data data; - struct rdt_domain *d; - char *dom = NULL, *id; - unsigned long dom_id; - -next: - if (!line || line[0] == '\0') - return 0; - - dom = strsep(&line, ";"); - id = strsep(&dom, "="); - if (!dom || kstrtoul(id, 10, &dom_id)) { - rdt_last_cmd_puts("Missing '=' or non-numeric domain\n"); - return -EINVAL; - } - - dom = strim(dom); - list_for_each_entry(d, &r->domains, list) { - if (d->id == dom_id) { - data.buf = dom; - data.mode = RDT_MODE_SHAREABLE; - data.closid = closid; - if (parse_cbm(&data, s, d)) - return -EINVAL; - /* - * Keep io_alloc CLOSID's CBM of CDP_CODE and CDP_DATA - * in sync. - */ - if (resctrl_arch_get_cdp_enabled(r->rid)) { - peer_type = resctrl_peer_type(s->conf_type); - memcpy(&d->staged_config[peer_type], - &d->staged_config[s->conf_type], - sizeof(d->staged_config[0])); - } - goto next; - } - } - - return -EINVAL; -} - -ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, loff_t off) -{ - struct resctrl_schema *s = of->kn->parent->priv; - struct rdt_resource *r = s->res; - u32 io_alloc_closid; - int ret = 0; - - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - - buf[nbytes - 1] = '\0'; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - rdt_last_cmd_clear(); - - if (!r->cache.io_alloc_capable) { - rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); - ret = -ENODEV; - goto out_unlock; - } - - if (!resctrl_arch_get_io_alloc_enabled(r)) { - rdt_last_cmd_printf("io_alloc is not enabled on %s\n", s->name); - ret = -EINVAL; - goto out_unlock; - } - - io_alloc_closid = resctrl_io_alloc_closid(r); - - rdt_staged_configs_clear(); - ret = resctrl_io_alloc_parse_line(buf, r, s, io_alloc_closid); - if (ret) - goto out_clear_configs; - - ret = resctrl_arch_update_domains(r, io_alloc_closid); - -out_clear_configs: - rdt_staged_configs_clear(); -out_unlock: - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); - - return ret ?: nbytes; -} -#endif diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index 3283448e59559a8137021ea493e22eaebbd7c33a..9a8cf6f11151d9c8512344505388135d4196c388 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -3,18 +3,11 @@ #define _FS_RESCTRL_INTERNAL_H #include -#include #include #include -#include #include -#include - -/* Maximum assignable counters per resctrl group */ -#define MAX_CNTRS 2 - -#define MON_CNTR_UNSET U32_MAX +#define CQM_LIMBOCHECK_INTERVAL 1000 /** * cpumask_any_housekeeping() - Choose any CPU in @mask, preferring those that @@ -31,30 +24,16 @@ static inline unsigned int cpumask_any_housekeeping(const struct cpumask *mask, int exclude_cpu) { - unsigned int cpu, hk_cpu; - - if (exclude_cpu == RESCTRL_PICK_ANY_CPU) - cpu = cpumask_any(mask); - else - cpu = cpumask_any_but(mask, exclude_cpu); - - /* Only continue if tick_nohz_full_mask has been initialized. */ - if (!tick_nohz_full_enabled()) - return cpu; - - /* If the CPU picked isn't marked nohz_full nothing more needs doing. */ - if (cpu < nr_cpu_ids && !tick_nohz_full_cpu(cpu)) - return cpu; + unsigned int cpu; /* Try to find a CPU that isn't nohz_full to use in preference */ - hk_cpu = cpumask_nth_andnot(0, mask, tick_nohz_full_mask); - if (hk_cpu == exclude_cpu) - hk_cpu = cpumask_nth_andnot(1, mask, tick_nohz_full_mask); - - if (hk_cpu < nr_cpu_ids) - cpu = hk_cpu; + if (tick_nohz_full_enabled()) { + cpu = cpumask_any_andnot_but(mask, tick_nohz_full_mask, exclude_cpu); + if (cpu < nr_cpu_ids) + return cpu; + } - return cpu; + return cpumask_any_but(mask, exclude_cpu); } struct rdt_fs_context { @@ -63,7 +42,6 @@ struct rdt_fs_context { bool enable_cdpl3; bool enable_mba_mbps; bool enable_debug; - bool enable_hwdrc_mb; }; static inline struct rdt_fs_context *rdt_fc2context(struct fs_context *fc) @@ -78,7 +56,7 @@ static inline struct rdt_fs_context *rdt_fc2context(struct fs_context *fc) * @evtid: event id * @name: name of the event * @configurable: true if the event is configurable - * @list: entry in &rdt_resource->mon.evt_list + * @list: entry in &rdt_resource->evt_list */ struct mon_evt { enum resctrl_event_id evtid; @@ -88,35 +66,60 @@ struct mon_evt { }; /** - * union mon_data_bits - Monitoring details for each event file - * @priv: Used to store monitoring event data in @u - * as kernfs private data - * @rid: Resource id associated with the event file - * @evtid: Event id associated with the event file - * @domid: The domain to which the event file belongs - * @u: Name of the bit fields struct + * struct mon_data - Monitoring details for each event file. + * @list: Member of the global @mon_data_kn_priv_list list. + * @rid: Resource id associated with the event file. + * @evtid: Event id associated with the event file. + * @sum: Set when event must be summed across multiple + * domains. + * @domid: When @sum is zero this is the domain to which + * the event file belongs. When @sum is one this + * is the id of the L3 cache that all domains to be + * summed share. + * + * Pointed to by the kernfs kn->priv field of monitoring event files. + * Readers and writers must hold rdtgroup_mutex. */ -union mon_data_bits { - void *priv; - struct { - unsigned int rid : 10; - enum resctrl_event_id evtid : 8; - unsigned int domid : 14; - } u; +struct mon_data { + struct list_head list; + enum resctrl_res_level rid; + enum resctrl_event_id evtid; + int domid; + bool sum; }; +/** + * struct rmid_read - Data passed across smp_call*() to read event count. + * @rgrp: Resource group for which the counter is being read. If it is a parent + * resource group then its event count is summed with the count from all + * its child resource groups. + * @r: Resource describing the properties of the event being read. + * @d: Domain that the counter should be read from. If NULL then sum all + * domains in @r sharing L3 @ci.id + * @evtid: Which monitor event to read. + * @first: Initialize MBM counter when true. + * @ci: Cacheinfo for L3. Only set when @d is NULL. Used when summing domains. + * @err: Error encountered when reading counter. + * @val: Returned value of event counter. If @rgrp is a parent resource group, + * @val includes the sum of event counts from its child resource groups. + * If @d is NULL, @val includes the sum of all domains in @r sharing @ci.id, + * (summed across child resource groups if @rgrp is a parent resource group). + * @arch_mon_ctx: Hardware monitor allocated for this read request (MPAM only). + */ struct rmid_read { struct rdtgroup *rgrp; struct rdt_resource *r; - struct rdt_domain *d; + struct rdt_mon_domain *d; enum resctrl_event_id evtid; bool first; + struct cacheinfo *ci; int err; u64 val; void *arch_mon_ctx; }; extern struct list_head resctrl_schema_all; + extern bool resctrl_mounted; enum rdt_group_type { @@ -160,14 +163,12 @@ enum rdtgrp_mode { * @parent: parent rdtgrp * @crdtgrp_list: child rdtgroup node list * @rmid: rmid for this rdtgroup - * @cntr_id: Counter ids for assignment */ struct mongroup { struct kernfs_node *mon_data_kn; struct rdtgroup *parent; struct list_head crdtgrp_list; u32 rmid; - u32 cntr_id[MAX_CNTRS]; }; /** @@ -183,6 +184,7 @@ struct mongroup { * monitor only or ctrl_mon group * @mon: mongroup related data * @mode: mode of resource group + * @mba_mbps_event: input monitoring event id when mba_sc is enabled * @plr: pseudo-locked region */ struct rdtgroup { @@ -195,13 +197,49 @@ struct rdtgroup { enum rdt_group_type type; struct mongroup mon; enum rdtgrp_mode mode; + enum resctrl_event_id mba_mbps_event; struct pseudo_lock_region *plr; }; +/* rdtgroup.flags */ +#define RDT_DELETED 1 + +/* rftype.flags */ +#define RFTYPE_FLAGS_CPUS_LIST 1 + +/* + * Define the file type flags for base and info directories. + */ +#define RFTYPE_INFO BIT(0) + +#define RFTYPE_BASE BIT(1) + +#define RFTYPE_CTRL BIT(4) + +#define RFTYPE_MON BIT(5) + +#define RFTYPE_TOP BIT(6) + +#define RFTYPE_RES_CACHE BIT(8) + +#define RFTYPE_RES_MB BIT(9) + +#define RFTYPE_DEBUG BIT(10) + +#define RFTYPE_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL) + +#define RFTYPE_MON_INFO (RFTYPE_INFO | RFTYPE_MON) + +#define RFTYPE_TOP_INFO (RFTYPE_INFO | RFTYPE_TOP) + +#define RFTYPE_CTRL_BASE (RFTYPE_BASE | RFTYPE_CTRL) + +#define RFTYPE_MON_BASE (RFTYPE_BASE | RFTYPE_MON) + /* List of all resource groups */ extern struct list_head rdt_all_groups; -extern int max_name_width, max_data_width; +extern int max_name_width; /** * struct rftype - describe each file in the resctrl file system @@ -241,114 +279,119 @@ struct mbm_state { u32 prev_bw; }; -static inline bool is_mba_sc(struct rdt_resource *r) -{ - if (!r) - r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); - - /* - * The software controller support is only applicable to MBA resource. - * Make sure to check for resource type. - */ - if (r->rid != RDT_RESOURCE_MBA) - return false; - - return r->membw.mba_sc; -} +extern struct mutex rdtgroup_mutex; -static inline bool is_hwdrc_enabled(struct rdt_resource *r) +static inline const char *rdt_kn_name(const struct kernfs_node *kn) { - if (!r) - r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); - - return r->membw.hwdrc_mb; + return rcu_dereference_check(kn->name, lockdep_is_held(&rdtgroup_mutex)); } -extern struct mutex rdtgroup_mutex; extern struct rdtgroup rdtgroup_default; + extern struct dentry *debugfs_resctrl; +extern enum resctrl_event_id mba_mbps_default_event; + void rdt_last_cmd_clear(void); + void rdt_last_cmd_puts(const char *s); + __printf(1, 2) void rdt_last_cmd_printf(const char *fmt, ...); struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn); + void rdtgroup_kn_unlock(struct kernfs_node *kn); + int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name); + int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name, umode_t mask); + ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); + int rdtgroup_schemata_show(struct kernfs_open_file *of, struct seq_file *s, void *v); -bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_domain *d, + +ssize_t rdtgroup_mba_mbps_event_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off); + +int rdtgroup_mba_mbps_event_show(struct kernfs_open_file *of, + struct seq_file *s, void *v); + +bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_ctrl_domain *d, unsigned long cbm, int closid, bool exclusive); -unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_domain *d, + +unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_ctrl_domain *d, unsigned long cbm); + enum rdtgrp_mode rdtgroup_mode_by_closid(int closid); + int rdtgroup_tasks_assigned(struct rdtgroup *r); + int closids_supported(void); + void closid_free(int closid); + int alloc_rmid(u32 closid); + void free_rmid(u32 closid, u32 rmid); + void resctrl_mon_resource_exit(void); + void mon_event_count(void *info); + int rdtgroup_mondata_show(struct seq_file *m, void *arg); + void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, - struct rdt_domain *d, struct rdtgroup *rdtgrp, - int evtid, int first); + struct rdt_mon_domain *d, struct rdtgroup *rdtgrp, + cpumask_t *cpumask, int evtid, int first); + int resctrl_mon_resource_init(void); -void mbm_setup_overflow_handler(struct rdt_domain *dom, + +void mbm_setup_overflow_handler(struct rdt_mon_domain *dom, unsigned long delay_ms, int exclude_cpu); + void mbm_handle_overflow(struct work_struct *work); + bool is_mba_sc(struct rdt_resource *r); -void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms, + +void cqm_setup_limbo_handler(struct rdt_mon_domain *dom, unsigned long delay_ms, int exclude_cpu); -void cqm_handle_limbo(struct work_struct *work); -bool has_busy_rmid(struct rdt_domain *d); -void __check_limbo(struct rdt_domain *d, bool force_free); -int mbm_cntr_alloc(struct rdt_resource *r); -void mbm_cntr_free(u32 cntr_id); -void rdt_staged_configs_clear(void); -bool closid_allocated(unsigned int closid); -bool closid_alloc_fixed(u32 closid); +void cqm_handle_limbo(struct work_struct *work); -int resctrl_find_cleanest_closid(void); -unsigned int mon_event_config_index_get(u32 evtid); -int rdtgroup_assign_cntr(struct rdtgroup *rdtgrp, enum resctrl_event_id evtid); -int rdtgroup_alloc_cntr(struct rdtgroup *rdtgrp, int index); -int rdtgroup_unassign_cntr(struct rdtgroup *rdtgrp, enum resctrl_event_id evtid); -void rdtgroup_free_cntr(struct rdt_resource *r, struct rdtgroup *rdtgrp, int index); +bool has_busy_rmid(struct rdt_mon_domain *d); -#ifdef CONFIG_X86 -int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, void *v); +void __check_limbo(struct rdt_mon_domain *d, bool force_free); -int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid); +void resctrl_file_fflags_init(const char *config, unsigned long fflags); -enum resctrl_conf_type resctrl_peer_type(enum resctrl_conf_type my_type); +void rdt_staged_configs_clear(void); -ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, loff_t off); +bool closid_allocated(unsigned int closid); -const char *rdtgroup_name_by_closid(u32 closid); -int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, - void *v); -ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, loff_t off); -#endif +int resctrl_find_cleanest_closid(void); #ifdef CONFIG_RESCTRL_FS_PSEUDO_LOCK int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp); + int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp); -bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm); -bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d); + +bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm); + +bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d); + int rdt_pseudo_lock_init(void); + void rdt_pseudo_lock_release(void); + int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp); + void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp); + #else static inline int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) { @@ -360,12 +403,12 @@ static inline int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) return -EOPNOTSUPP; } -static inline bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm) +static inline bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm) { return false; } -static inline bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) +static inline bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d) { return false; } diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index b6bb310bb10bdc6b7eeb3cb652d4847b72065e68..7326c28a7908f31660e5233afb090656dedd1057 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -15,13 +15,20 @@ * Software Developer Manual June 2016, volume 3, section 17.17. */ +#define pr_fmt(fmt) "resctrl: " fmt + #include -#include +#include #include #include + #include "internal.h" -/* +#define CREATE_TRACE_POINTS + +#include "monitor_trace.h" + +/** * struct rmid_entry - dirty tracking for all RMID. * @closid: The CLOSID for this entry. * @rmid: The RMID for this entry. @@ -123,7 +130,7 @@ static void limbo_release_entry(struct rmid_entry *entry) * decrement the count. If the busy count gets to zero on an RMID, we * free the RMID */ -void __check_limbo(struct rdt_domain *d, bool force_free) +void __check_limbo(struct rdt_mon_domain *d, bool force_free) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); @@ -158,6 +165,16 @@ void __check_limbo(struct rdt_domain *d, bool force_free) rmid_dirty = true; } else { rmid_dirty = (val >= resctrl_rmid_realloc_threshold); + + /* + * x86's CLOSID and RMID are independent numbers, so the entry's + * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the + * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't + * used to select the configuration. It is thus necessary to track both + * CLOSID and RMID because there may be dependencies between them + * on some architectures. + */ + trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val); } if (force_free || !rmid_dirty) { @@ -171,7 +188,7 @@ void __check_limbo(struct rdt_domain *d, bool force_free) resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx); } -bool has_busy_rmid(struct rdt_domain *d) +bool has_busy_rmid(struct rdt_mon_domain *d) { u32 idx_limit = resctrl_arch_system_num_rmid_idx(); @@ -272,7 +289,7 @@ int alloc_rmid(u32 closid) static void add_rmid_to_limbo(struct rmid_entry *entry) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - struct rdt_domain *d; + struct rdt_mon_domain *d; u32 idx; lockdep_assert_held(&rdtgroup_mutex); @@ -283,7 +300,7 @@ static void add_rmid_to_limbo(struct rmid_entry *entry) idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid); entry->busy = 0; - list_for_each_entry(d, &r->domains, list) { + list_for_each_entry(d, &r->mon_domains, hdr.list) { /* * For the first limbo RMID in the domain, * setup up the limbo worker. @@ -325,7 +342,7 @@ void free_rmid(u32 closid, u32 rmid) list_add_tail(&entry->list, &rmid_free_lru); } -static struct mbm_state *get_mbm_state(struct rdt_domain *d, u32 closid, +static struct mbm_state *get_mbm_state(struct rdt_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id evtid) { u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); @@ -342,7 +359,10 @@ static struct mbm_state *get_mbm_state(struct rdt_domain *d, u32 closid, static int __mon_event_count(u32 closid, u32 rmid, struct rmid_read *rr) { + int cpu = smp_processor_id(); + struct rdt_mon_domain *d; struct mbm_state *m; + int err, ret; u64 tval = 0; if (rr->first) { @@ -353,14 +373,47 @@ static int __mon_event_count(u32 closid, u32 rmid, struct rmid_read *rr) return 0; } - rr->err = resctrl_arch_rmid_read(rr->r, rr->d, closid, rmid, rr->evtid, - &tval, rr->arch_mon_ctx); - if (rr->err) - return rr->err; + if (rr->d) { + /* Reading a single domain, must be on a CPU in that domain. */ + if (!cpumask_test_cpu(cpu, &rr->d->hdr.cpu_mask)) + return -EINVAL; + rr->err = resctrl_arch_rmid_read(rr->r, rr->d, closid, rmid, + rr->evtid, &tval, rr->arch_mon_ctx); + if (rr->err) + return rr->err; - rr->val += tval; + rr->val += tval; - return 0; + return 0; + } + + /* Summing domains that share a cache, must be on a CPU for that cache. */ + if (!cpumask_test_cpu(cpu, &rr->ci->shared_cpu_map)) + return -EINVAL; + + /* + * Legacy files must report the sum of an event across all + * domains that share the same L3 cache instance. + * Report success if a read from any domain succeeds, -EINVAL + * (translated to "Unavailable" for user space) if reading from + * all domains fail for any reason. + */ + ret = -EINVAL; + list_for_each_entry(d, &rr->r->mon_domains, hdr.list) { + if (d->ci_id != rr->ci->id) + continue; + err = resctrl_arch_rmid_read(rr->r, d, closid, rmid, + rr->evtid, &tval, rr->arch_mon_ctx); + if (!err) { + rr->val += tval; + ret = 0; + } + } + + if (ret) + rr->err = ret; + + return ret; } /* @@ -377,9 +430,12 @@ static int __mon_event_count(u32 closid, u32 rmid, struct rmid_read *rr) */ static void mbm_bw_count(u32 closid, u32 rmid, struct rmid_read *rr) { - u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); - struct mbm_state *m = &rr->d->mbm_local[idx]; u64 cur_bw, bytes, cur_bytes; + struct mbm_state *m; + + m = get_mbm_state(rr->d, closid, rmid, rr->evtid); + if (WARN_ON_ONCE(!m)) + return; cur_bytes = rr->val; bytes = cur_bytes - m->prev_bw_bytes; @@ -429,6 +485,22 @@ void mon_event_count(void *info) rr->err = 0; } +static struct rdt_ctrl_domain *get_ctrl_domain_from_cpu(int cpu, + struct rdt_resource *r) +{ + struct rdt_ctrl_domain *d; + + lockdep_assert_cpus_held(); + + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + /* Find the domain that contains this CPU */ + if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) + return d; + } + + return NULL; +} + /* * Feedback loop for MBA software controller (mba_sc) * @@ -461,27 +533,27 @@ void mon_event_count(void *info) * throttle MSRs already have low percentage values. To avoid * unnecessarily restricting such rdtgroups, we also increase the bandwidth. */ -static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) +static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_mon_domain *dom_mbm) { u32 closid, rmid, cur_msr_val, new_msr_val; struct mbm_state *pmbm_data, *cmbm_data; + struct rdt_ctrl_domain *dom_mba; + enum resctrl_event_id evt_id; struct rdt_resource *r_mba; - struct rdt_domain *dom_mba; - u32 cur_bw, user_bw, idx; struct list_head *head; struct rdtgroup *entry; - - if (!resctrl_arch_is_mbm_local_enabled()) - return; + u32 cur_bw, user_bw; r_mba = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + evt_id = rgrp->mba_mbps_event; closid = rgrp->closid; rmid = rgrp->mon.rmid; - idx = resctrl_arch_rmid_idx_encode(closid, rmid); - pmbm_data = &dom_mbm->mbm_local[idx]; + pmbm_data = get_mbm_state(dom_mbm, closid, rmid, evt_id); + if (WARN_ON_ONCE(!pmbm_data)) + return; - dom_mba = resctrl_get_domain_from_cpu(smp_processor_id(), r_mba); + dom_mba = get_ctrl_domain_from_cpu(smp_processor_id(), r_mba); if (!dom_mba) { pr_warn_once("Failure to get domain for MBA update\n"); return; @@ -498,7 +570,9 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) */ head = &rgrp->mon.crdtgrp_list; list_for_each_entry(entry, head, mon.crdtgrp_list) { - cmbm_data = &dom_mbm->mbm_local[entry->mon.rmid]; + cmbm_data = get_mbm_state(dom_mbm, entry->closid, entry->mon.rmid, evt_id); + if (WARN_ON_ONCE(!cmbm_data)) + return; cur_bw += cmbm_data->prev_bw; } @@ -527,55 +601,45 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) resctrl_arch_update_one(r_mba, dom_mba, closid, CDP_NONE, new_msr_val); } -static void mbm_update(struct rdt_resource *r, struct rdt_domain *d, - u32 closid, u32 rmid) +static void mbm_update_one_event(struct rdt_resource *r, struct rdt_mon_domain *d, + u32 closid, u32 rmid, enum resctrl_event_id evtid) { - struct rmid_read rr; + struct rmid_read rr = {0}; - rr.first = false; rr.r = r; rr.d = d; + rr.evtid = evtid; + rr.arch_mon_ctx = resctrl_arch_mon_ctx_alloc(rr.r, rr.evtid); + if (IS_ERR(rr.arch_mon_ctx)) { + pr_warn_ratelimited("Failed to allocate monitor context: %ld", + PTR_ERR(rr.arch_mon_ctx)); + return; + } + + __mon_event_count(closid, rmid, &rr); /* - * This is protected from concurrent reads from user - * as both the user and we hold the global mutex. + * If the software controller is enabled, compute the + * bandwidth for this event id. */ - if (resctrl_arch_is_mbm_total_enabled()) { - rr.evtid = QOS_L3_MBM_TOTAL_EVENT_ID; - rr.val = 0; - rr.arch_mon_ctx = resctrl_arch_mon_ctx_alloc(rr.r, rr.evtid); - if (IS_ERR(rr.arch_mon_ctx)) { - pr_warn_ratelimited("Failed to allocate monitor context: %ld", - PTR_ERR(rr.arch_mon_ctx)); - return; - } - - __mon_event_count(closid, rmid, &rr); - - resctrl_arch_mon_ctx_free(rr.r, rr.evtid, rr.arch_mon_ctx); - } - if (resctrl_arch_is_mbm_local_enabled()) { - rr.evtid = QOS_L3_MBM_LOCAL_EVENT_ID; - rr.val = 0; - rr.arch_mon_ctx = resctrl_arch_mon_ctx_alloc(rr.r, rr.evtid); - if (IS_ERR(rr.arch_mon_ctx)) { - pr_warn_ratelimited("Failed to allocate monitor context: %ld", - PTR_ERR(rr.arch_mon_ctx)); - return; - } + if (is_mba_sc(NULL)) + mbm_bw_count(closid, rmid, &rr); - __mon_event_count(closid, rmid, &rr); + resctrl_arch_mon_ctx_free(rr.r, rr.evtid, rr.arch_mon_ctx); +} - /* - * Call the MBA software controller only for the - * control groups and when user has enabled - * the software controller explicitly. - */ - if (is_mba_sc(NULL)) - mbm_bw_count(closid, rmid, &rr); +static void mbm_update(struct rdt_resource *r, struct rdt_mon_domain *d, + u32 closid, u32 rmid) +{ + /* + * This is protected from concurrent reads from user as both + * the user and overflow handler hold the global mutex. + */ + if (resctrl_arch_is_mbm_total_enabled()) + mbm_update_one_event(r, d, closid, rmid, QOS_L3_MBM_TOTAL_EVENT_ID); - resctrl_arch_mon_ctx_free(rr.r, rr.evtid, rr.arch_mon_ctx); - } + if (resctrl_arch_is_mbm_local_enabled()) + mbm_update_one_event(r, d, closid, rmid, QOS_L3_MBM_LOCAL_EVENT_ID); } /* @@ -585,17 +649,17 @@ static void mbm_update(struct rdt_resource *r, struct rdt_domain *d, void cqm_handle_limbo(struct work_struct *work) { unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL); - struct rdt_domain *d; + struct rdt_mon_domain *d; cpus_read_lock(); mutex_lock(&rdtgroup_mutex); - d = container_of(work, struct rdt_domain, cqm_limbo.work); + d = container_of(work, struct rdt_mon_domain, cqm_limbo.work); __check_limbo(d, false); if (has_busy_rmid(d)) { - d->cqm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask, + d->cqm_work_cpu = cpumask_any_housekeeping(&d->hdr.cpu_mask, RESCTRL_PICK_ANY_CPU); schedule_delayed_work_on(d->cqm_work_cpu, &d->cqm_limbo, delay); @@ -613,36 +677,26 @@ void cqm_handle_limbo(struct work_struct *work) * @exclude_cpu: Which CPU the handler should not run on, * RESCTRL_PICK_ANY_CPU to pick any CPU. */ -void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms, +void cqm_setup_limbo_handler(struct rdt_mon_domain *dom, unsigned long delay_ms, int exclude_cpu) { unsigned long delay = msecs_to_jiffies(delay_ms); int cpu; - cpu = cpumask_any_housekeeping(&dom->cpu_mask, exclude_cpu); + cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu); dom->cqm_work_cpu = cpu; if (cpu < nr_cpu_ids) schedule_delayed_work_on(cpu, &dom->cqm_limbo, delay); } -bool is_rdt_domain_valid(struct rdt_resource *r, struct rdt_domain *d) -{ - int i; - - for (i = 0; i < NR_CPUS; i++) - if (r->rdt_domain_list[i] == d) - return true; - return false; -} - void mbm_handle_overflow(struct work_struct *work) { unsigned long delay = msecs_to_jiffies(MBM_OVERFLOW_INTERVAL); struct rdtgroup *prgrp, *crgrp; + struct rdt_mon_domain *d; struct list_head *head; struct rdt_resource *r; - struct rdt_domain *d; cpus_read_lock(); mutex_lock(&rdtgroup_mutex); @@ -655,10 +709,7 @@ void mbm_handle_overflow(struct work_struct *work) goto out_unlock; r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - d = container_of(work, struct rdt_domain, mbm_over.work); - - if (!is_rdt_domain_valid(r, d)) - goto out_unlock; + d = container_of(work, struct rdt_mon_domain, mbm_over.work); list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { mbm_update(r, d, prgrp->closid, prgrp->mon.rmid); @@ -675,7 +726,7 @@ void mbm_handle_overflow(struct work_struct *work) * Re-check for housekeeping CPUs. This allows the overflow handler to * move off a nohz_full CPU quickly. */ - d->mbm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask, + d->mbm_work_cpu = cpumask_any_housekeeping(&d->hdr.cpu_mask, RESCTRL_PICK_ANY_CPU); schedule_delayed_work_on(d->mbm_work_cpu, &d->mbm_over, delay); @@ -692,7 +743,7 @@ void mbm_handle_overflow(struct work_struct *work) * @exclude_cpu: Which CPU the handler should not run on, * RESCTRL_PICK_ANY_CPU to pick any CPU. */ -void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms, +void mbm_setup_overflow_handler(struct rdt_mon_domain *dom, unsigned long delay_ms, int exclude_cpu) { unsigned long delay = msecs_to_jiffies(delay_ms); @@ -704,7 +755,7 @@ void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms, */ if (!resctrl_mounted || !resctrl_arch_mon_capable()) return; - cpu = cpumask_any_housekeeping(&dom->cpu_mask, exclude_cpu); + cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu); dom->mbm_work_cpu = cpu; if (cpu < nr_cpu_ids) @@ -759,7 +810,7 @@ static int dom_data_init(struct rdt_resource *r) /* * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are special and * are always allocated. These are used for the rdtgroup_default - * control group, which will be setup later in rdtgroup_init(). + * control group, which will be setup later in resctrl_init(). */ idx = resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID, RESCTRL_RESERVED_RMID); @@ -774,10 +825,11 @@ static int dom_data_init(struct rdt_resource *r) static void dom_data_exit(struct rdt_resource *r) { + mutex_lock(&rdtgroup_mutex); + if (!r->mon_capable) - return; + goto out_unlock; - mutex_lock(&rdtgroup_mutex); if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { kfree(closid_num_dirty_rmid); closid_num_dirty_rmid = NULL; @@ -786,6 +838,7 @@ static void dom_data_exit(struct rdt_resource *r) kfree(rmid_ptrs); rmid_ptrs = NULL; +out_unlock: mutex_unlock(&rdtgroup_mutex); } @@ -804,11 +857,6 @@ static struct mon_evt mbm_local_event = { .evtid = QOS_L3_MBM_LOCAL_EVENT_ID, }; -static struct mon_evt mbm_bps_event = { - .name = "mbm_local_bytes", - .evtid = QOS_MC_MBM_BPS_EVENT_ID, -}; - /* * Initialize the event list for the resource. * @@ -818,24 +866,28 @@ static struct mon_evt mbm_bps_event = { */ static void l3_mon_evt_init(struct rdt_resource *r) { - INIT_LIST_HEAD(&r->mon.evt_list); + INIT_LIST_HEAD(&r->evt_list); if (resctrl_arch_is_llc_occupancy_enabled()) - list_add_tail(&llc_occupancy_event.list, &r->mon.evt_list); + list_add_tail(&llc_occupancy_event.list, &r->evt_list); if (resctrl_arch_is_mbm_total_enabled()) - list_add_tail(&mbm_total_event.list, &r->mon.evt_list); + list_add_tail(&mbm_total_event.list, &r->evt_list); if (resctrl_arch_is_mbm_local_enabled()) - list_add_tail(&mbm_local_event.list, &r->mon.evt_list); -} - -static void mc_mon_evt_init(struct rdt_resource *r) -{ - INIT_LIST_HEAD(&r->mon.evt_list); - - if (resctrl_arch_is_mbm_bps_enabled()) - list_add_tail(&mbm_bps_event.list, &r->mon.evt_list); + list_add_tail(&mbm_local_event.list, &r->evt_list); } +/** + * resctrl_mon_resource_init() - Initialise global monitoring structures. + * + * Allocate and initialise global monitor resources that do not belong to a + * specific domain. i.e. the rmid_ptrs[] used for the limbo and free lists. + * Called once during boot after the struct rdt_resource's have been configured + * but before the filesystem is mounted. + * Resctrl's cpuhp callbacks may be called before this point to bring a domain + * online. + * + * Returns 0 for success, or -ENOMEM. + */ int resctrl_mon_resource_init(void) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); @@ -861,8 +913,10 @@ int resctrl_mon_resource_init(void) RFTYPE_MON_INFO | RFTYPE_RES_CACHE); } - r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); - mc_mon_evt_init(r); + if (resctrl_arch_is_mbm_local_enabled()) + mba_mbps_default_event = QOS_L3_MBM_LOCAL_EVENT_ID; + else if (resctrl_arch_is_mbm_total_enabled()) + mba_mbps_default_event = QOS_L3_MBM_TOTAL_EVENT_ID; return 0; } diff --git a/fs/resctrl/monitor_trace.h b/fs/resctrl/monitor_trace.h new file mode 100644 index 0000000000000000000000000000000000000000..fdf49f22576a9b6825d3ade8178eacf99e951256 --- /dev/null +++ b/fs/resctrl/monitor_trace.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM resctrl + +#if !defined(_FS_RESCTRL_MONITOR_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _FS_RESCTRL_MONITOR_TRACE_H + +#include + +TRACE_EVENT(mon_llc_occupancy_limbo, + TP_PROTO(u32 ctrl_hw_id, u32 mon_hw_id, int domain_id, u64 llc_occupancy_bytes), + TP_ARGS(ctrl_hw_id, mon_hw_id, domain_id, llc_occupancy_bytes), + TP_STRUCT__entry(__field(u32, ctrl_hw_id) + __field(u32, mon_hw_id) + __field(int, domain_id) + __field(u64, llc_occupancy_bytes)), + TP_fast_assign(__entry->ctrl_hw_id = ctrl_hw_id; + __entry->mon_hw_id = mon_hw_id; + __entry->domain_id = domain_id; + __entry->llc_occupancy_bytes = llc_occupancy_bytes;), + TP_printk("ctrl_hw_id=%u mon_hw_id=%u domain_id=%d llc_occupancy_bytes=%llu", + __entry->ctrl_hw_id, __entry->mon_hw_id, __entry->domain_id, + __entry->llc_occupancy_bytes) + ); + +#endif /* _FS_RESCTRL_MONITOR_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#define TRACE_INCLUDE_FILE monitor_trace + +#include diff --git a/fs/resctrl/psuedo_lock.c b/fs/resctrl/pseudo_lock.c similarity index 93% rename from fs/resctrl/psuedo_lock.c rename to fs/resctrl/pseudo_lock.c index 077c2abb6edd90cad1e1e237dd4c1861cf1dc025..6a358fc7124d950cdad10f6cc1bc7329e589fd7e 100644 --- a/fs/resctrl/psuedo_lock.c +++ b/fs/resctrl/pseudo_lock.c @@ -17,15 +17,11 @@ #include #include #include -#include #include +#include #include #include -#include -#include -#include - #include "internal.h" /* @@ -33,6 +29,7 @@ * pseudo-locked regions. */ static unsigned int pseudo_lock_major; + static unsigned long pseudo_lock_minor_avail = GENMASK(MINORBITS, 0); static char *pseudo_lock_devnode(const struct device *dev, umode_t *mode) @@ -42,7 +39,8 @@ static char *pseudo_lock_devnode(const struct device *dev, umode_t *mode) rdtgrp = dev_get_drvdata(dev); if (mode) *mode = 0600; - return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name); + guard(mutex)(&rdtgroup_mutex); + return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdt_kn_name(rdtgrp->kn)); } static const struct class pseudo_lock_class = { @@ -155,7 +153,7 @@ static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr) int cpu; int ret; - for_each_cpu(cpu, &plr->d->cpu_mask) { + for_each_cpu(cpu, &plr->d->hdr.cpu_mask) { pm_req = kzalloc(sizeof(*pm_req), GFP_KERNEL); if (!pm_req) { rdt_last_cmd_puts("Failure to allocate memory for PM QoS\n"); @@ -226,12 +224,15 @@ static void pseudo_lock_region_clear(struct pseudo_lock_region *plr) */ static int pseudo_lock_region_init(struct pseudo_lock_region *plr) { - struct cpu_cacheinfo *ci; + enum resctrl_scope scope = plr->s->res->ctrl_scope; + struct cacheinfo *ci; int ret; - int i; + + if (WARN_ON_ONCE(scope != RESCTRL_L2_CACHE && scope != RESCTRL_L3_CACHE)) + return -ENODEV; /* Pick the first cpu we find that is associated with the cache. */ - plr->cpu = cpumask_first(&plr->d->cpu_mask); + plr->cpu = cpumask_first(&plr->d->hdr.cpu_mask); if (!cpu_online(plr->cpu)) { rdt_last_cmd_printf("CPU %u associated with cache not online\n", @@ -240,15 +241,11 @@ static int pseudo_lock_region_init(struct pseudo_lock_region *plr) goto out_region; } - ci = get_cpu_cacheinfo(plr->cpu); - - plr->size = rdtgroup_cbm_to_size(plr->s->res, plr->d, plr->cbm); - - for (i = 0; i < ci->num_leaves; i++) { - if (ci->info_list[i].level == plr->s->res->cache_level) { - plr->line_size = ci->info_list[i].coherency_line_size; - return 0; - } + ci = get_cpu_cacheinfo_level(plr->cpu, scope); + if (ci) { + plr->line_size = ci->coherency_line_size; + plr->size = rdtgroup_cbm_to_size(plr->s->res, plr->d, plr->cbm); + return 0; } ret = -1; @@ -614,7 +611,7 @@ int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) * Return: true if @cbm overlaps with pseudo-locked region on @d, false * otherwise. */ -bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm) +bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm) { unsigned int cbm_len; unsigned long cbm_b; @@ -641,12 +638,11 @@ bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm * if it is not possible to test due to memory allocation issue, * false otherwise. */ -bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) +bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d) { + struct rdt_ctrl_domain *d_i; cpumask_var_t cpu_with_psl; - enum resctrl_res_level i; struct rdt_resource *r; - struct rdt_domain *d_i; bool ret = false; /* Walking r->domains, ensure it can't race with cpuhp */ @@ -659,15 +655,11 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) * First determine which cpus have pseudo-locked regions * associated with them. */ - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - r = resctrl_arch_get_resource(i); - if (!r->alloc_capable) - continue; - - list_for_each_entry(d_i, &r->domains, list) { + for_each_alloc_capable_rdt_resource(r) { + list_for_each_entry(d_i, &r->ctrl_domains, hdr.list) { if (d_i->plr) cpumask_or(cpu_with_psl, cpu_with_psl, - &d_i->cpu_mask); + &d_i->hdr.cpu_mask); } } @@ -675,7 +667,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) * Next test if new pseudo-locked region would intersect with * existing region. */ - if (cpumask_intersects(&d->cpu_mask, cpu_with_psl)) + if (cpumask_intersects(&d->hdr.cpu_mask, cpu_with_psl)) ret = true; free_cpumask_var(cpu_with_psl); @@ -715,7 +707,7 @@ static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) } plr->thread_done = 0; - cpu = cpumask_first(&plr->d->cpu_mask); + cpu = cpumask_first(&plr->d->hdr.cpu_mask); if (!cpu_online(cpu)) { ret = -ENODEV; goto out; @@ -724,20 +716,14 @@ static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) plr->cpu = cpu; if (sel == 1) - thread = kthread_create_on_node(resctrl_arch_measure_cycles_lat_fn, - plr, cpu_to_node(cpu), - "pseudo_lock_measure/%u", - cpu); + thread = kthread_run_on_cpu(resctrl_arch_measure_cycles_lat_fn, + plr, cpu, "pseudo_lock_measure/%u"); else if (sel == 2) - thread = kthread_create_on_node(resctrl_arch_measure_l2_residency, - plr, cpu_to_node(cpu), - "pseudo_lock_measure/%u", - cpu); + thread = kthread_run_on_cpu(resctrl_arch_measure_l2_residency, + plr, cpu, "pseudo_lock_measure/%u"); else if (sel == 3) - thread = kthread_create_on_node(resctrl_arch_measure_l3_residency, - plr, cpu_to_node(cpu), - "pseudo_lock_measure/%u", - cpu); + thread = kthread_run_on_cpu(resctrl_arch_measure_l3_residency, + plr, cpu, "pseudo_lock_measure/%u"); else goto out; @@ -745,8 +731,6 @@ static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) ret = PTR_ERR(thread); goto out; } - kthread_bind(thread, cpu); - wake_up_process(thread); ret = wait_event_interruptible(plr->lock_thread_wq, plr->thread_done == 1); @@ -780,13 +764,9 @@ static ssize_t pseudo_lock_measure_trigger(struct file *file, if (ret == 0) { if (sel != 1 && sel != 2 && sel != 3) return -EINVAL; - ret = debugfs_file_get(file->f_path.dentry); - if (ret) - return ret; ret = pseudo_lock_measure_cycles(rdtgrp, sel); if (ret == 0) ret = count; - debugfs_file_put(file->f_path.dentry); } return ret; @@ -820,6 +800,7 @@ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) struct task_struct *thread; unsigned int new_minor; struct device *dev; + char *kn_name __free(kfree) = NULL; int ret; ret = pseudo_lock_region_alloc(plr); @@ -831,22 +812,22 @@ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) ret = -EINVAL; goto out_region; } + kn_name = kstrdup(rdt_kn_name(rdtgrp->kn), GFP_KERNEL); + if (!kn_name) { + ret = -ENOMEM; + goto out_cstates; + } plr->thread_done = 0; - plr->closid = rdtgrp->closid; - thread = kthread_create_on_node(resctrl_arch_pseudo_lock_fn, plr, - cpu_to_node(plr->cpu), - "pseudo_lock/%u", plr->cpu); + thread = kthread_run_on_cpu(resctrl_arch_pseudo_lock_fn, plr, + plr->cpu, "pseudo_lock/%u"); if (IS_ERR(thread)) { ret = PTR_ERR(thread); rdt_last_cmd_printf("Locking thread returned error %d\n", ret); goto out_cstates; } - kthread_bind(thread, plr->cpu); - wake_up_process(thread); - ret = wait_event_interruptible(plr->lock_thread_wq, plr->thread_done == 1); if (ret < 0) { @@ -880,8 +861,7 @@ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) mutex_unlock(&rdtgroup_mutex); if (!IS_ERR_OR_NULL(debugfs_resctrl)) { - plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name, - debugfs_resctrl); + plr->debugfs_dir = debugfs_create_dir(kn_name, debugfs_resctrl); if (!IS_ERR_OR_NULL(plr->debugfs_dir)) debugfs_create_file("pseudo_lock_measure", 0200, plr->debugfs_dir, rdtgrp, @@ -890,7 +870,7 @@ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) dev = device_create(&pseudo_lock_class, NULL, MKDEV(pseudo_lock_major, new_minor), - rdtgrp, "%s", rdtgrp->kn->name); + rdtgrp, "%s", kn_name); mutex_lock(&rdtgroup_mutex); @@ -1046,7 +1026,7 @@ static int pseudo_lock_dev_mmap(struct file *filp, struct vm_area_struct *vma) * may be scheduled elsewhere and invalidate entries in the * pseudo-locked region. */ - if (!cpumask_subset(current->cpus_ptr, &plr->d->cpu_mask)) { + if (!cpumask_subset(current->cpus_ptr, &plr->d->hdr.cpu_mask)) { mutex_unlock(&rdtgroup_mutex); return -EINVAL; } diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 457de2adc251851a0d1286d2989aa487cc61b02f..31377854de1577a819e14550b9074fc9a131afd8 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -12,36 +12,41 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include #include #include #include #include #include #include +#include #include #include -#include #include #include -#include #include #include -#include #include "internal.h" /* Mutex to protect rdtgroup access. */ DEFINE_MUTEX(rdtgroup_mutex); static struct kernfs_root *rdt_root; + struct rdtgroup rdtgroup_default; + LIST_HEAD(rdt_all_groups); /* list of entries for the schemata file */ LIST_HEAD(resctrl_schema_all); +/* + * List of struct mon_data containing private data of event files for use by + * rdtgroup_mondata_show(). Protected by rdtgroup_mutex. + */ +static LIST_HEAD(mon_data_kn_priv_list); + /* The filesystem can only be mounted once. */ bool resctrl_mounted; @@ -55,19 +60,30 @@ static struct kernfs_node *kn_mongrp; static struct kernfs_node *kn_mondata; /* - * Used to store the max resource name width and max resource data width - * to display the schemata in a tabular format + * Used to store the max resource name width to display the schemata names in + * a tabular format. */ -int max_name_width, max_data_width; +int max_name_width; static struct seq_buf last_cmd_status; + static char last_cmd_status_buf[512]; static int rdtgroup_setup_root(struct rdt_fs_context *ctx); + static void rdtgroup_destroy_root(void); struct dentry *debugfs_resctrl; +/* + * Memory bandwidth monitoring event to use for the default CTRL_MON group + * and each new CTRL_MON group created by the user. Only relevant when + * the filesystem is mounted with the "mba_MBps" option so it does not + * matter that it remains uninitialized on systems that do not support + * the "mba_MBps" option. + */ +enum resctrl_event_id mba_mbps_default_event; + static bool resctrl_debug; void rdt_last_cmd_clear(void) @@ -94,18 +110,13 @@ void rdt_last_cmd_printf(const char *fmt, ...) void rdt_staged_configs_clear(void) { - enum resctrl_res_level i; + struct rdt_ctrl_domain *dom; struct rdt_resource *r; - struct rdt_domain *dom; lockdep_assert_held(&rdtgroup_mutex); - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - r = resctrl_arch_get_resource(i); - if (!r->alloc_capable) - continue; - - list_for_each_entry(dom, &r->domains, list) + for_each_alloc_capable_rdt_resource(r) { + list_for_each_entry(dom, &r->ctrl_domains, hdr.list) memset(dom->staged_config, 0, sizeof(dom->staged_config)); } } @@ -138,6 +149,7 @@ static bool resctrl_is_mbm_event(int e) * limited as the number of resources grows. */ static unsigned long *closid_free_map; + static int closid_free_map_len; int closids_supported(void) @@ -145,21 +157,35 @@ int closids_supported(void) return closid_free_map_len; } -static void closid_init(void) +static int closid_init(void) { struct resctrl_schema *s; u32 rdt_min_closid = ~0; + /* Monitor only platforms still call closid_init() */ + if (list_empty(&resctrl_schema_all)) + return 0; + /* Compute rdt_min_closid across all resources */ list_for_each_entry(s, &resctrl_schema_all, list) rdt_min_closid = min(rdt_min_closid, s->num_closid); closid_free_map = bitmap_alloc(rdt_min_closid, GFP_KERNEL); + if (!closid_free_map) + return -ENOMEM; bitmap_fill(closid_free_map, rdt_min_closid); /* RESCTRL_RESERVED_CLOSID is always reserved for the default group */ __clear_bit(RESCTRL_RESERVED_CLOSID, closid_free_map); closid_free_map_len = rdt_min_closid; + + return 0; +} + +static void closid_exit(void) +{ + bitmap_free(closid_free_map); + closid_free_map = NULL; } static int closid_alloc(void) @@ -180,7 +206,6 @@ static int closid_alloc(void) if (closid == closid_free_map_len) return -ENOSPC; } - __clear_bit(closid, closid_free_map); return closid; @@ -207,42 +232,6 @@ bool closid_allocated(unsigned int closid) return !test_bit(closid, closid_free_map); } -/* - * Counter bitmap for tracking the available counters. - * ABMC feature provides set of hardware counters for enabling events. - * Each event takes one hardware counter. Kernel needs to keep track - * of number of available counters. - */ -static DECLARE_BITMAP(mbm_cntrs_free_map, 64); - -static void mbm_cntrs_init(struct rdt_resource *r) -{ - bitmap_fill(mbm_cntrs_free_map, r->mon.num_mbm_cntrs); -} - -int mbm_cntr_alloc(struct rdt_resource *r) -{ - int cntr_id; - - cntr_id = find_first_bit(mbm_cntrs_free_map, r->mon.num_mbm_cntrs); - if (cntr_id >= r->mon.num_mbm_cntrs) - return -ENOSPC; - - __clear_bit(cntr_id, mbm_cntrs_free_map); - - return cntr_id; -} - -void mbm_cntr_free(u32 cntr_id) -{ - __set_bit(cntr_id, mbm_cntrs_free_map); -} - -bool closid_alloc_fixed(u32 closid) -{ - return __test_and_clear_bit(closid, closid_free_map); -} - /** * rdtgroup_mode_by_closid - Return mode of resource group with closid * @closid: closid if the resource group @@ -374,7 +363,7 @@ static int rdtgroup_cpus_show(struct kernfs_open_file *of, rdt_last_cmd_puts("Cache domain offline\n"); ret = -ENODEV; } else { - mask = &rdtgrp->plr->d->cpu_mask; + mask = &rdtgrp->plr->d->hdr.cpu_mask; seq_printf(s, is_cpu_list(of) ? "%*pbl\n" : "%*pb\n", cpumask_pr_args(mask)); @@ -400,17 +389,15 @@ static int rdtgroup_cpus_show(struct kernfs_open_file *of, static void update_closid_rmid(const struct cpumask *cpu_mask, struct rdtgroup *r) { - struct resctrl_cpu_sync defaults; - struct resctrl_cpu_sync *defaults_p = NULL; + struct resctrl_cpu_defaults defaults, *p = NULL; if (r) { defaults.closid = r->closid; defaults.rmid = r->mon.rmid; - defaults_p = &defaults; + p = &defaults; } - on_each_cpu_mask(cpu_mask, resctrl_arch_sync_cpu_defaults, defaults_p, - 1); + on_each_cpu_mask(cpu_mask, resctrl_arch_sync_cpu_closid_rmid, p, 1); } static int cpus_mon_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask, @@ -549,6 +536,8 @@ static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, goto unlock; } + rdt_last_cmd_clear(); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { ret = -EINVAL; @@ -891,477 +880,7 @@ static int rdtgroup_rmid_show(struct kernfs_open_file *of, return ret; } -static int rdtgroup_mbm_mode_show(struct kernfs_open_file *of, - struct seq_file *s, void *v) -{ - struct rdt_resource *r = of->kn->parent->priv; - - if (r->mon.mbm_cntr_assignable) { - if (resctrl_arch_get_abmc_enabled()) { - seq_puts(s, "[mbm_cntr_assign]\n"); - seq_puts(s, "legacy\n"); - } else { - seq_puts(s, "mbm_cntr_assign\n"); - seq_puts(s, "[legacy]\n"); - } - } else { - seq_puts(s, "[legacy]\n"); - } - - return 0; -} - -static void rdtgroup_mbm_cntr_reset(struct rdt_resource *r) -{ - struct rdtgroup *prgrp, *crgrp; - struct rdt_domain *dom; - - mbm_cntrs_init(r); - - list_for_each_entry(dom, &r->domains, list) - bitmap_zero(dom->mbm_cntr_map, r->mon.num_mbm_cntrs); - - /* Reset the cntr_id's for all the monitor groups */ - list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { - prgrp->mon.cntr_id[0] = MON_CNTR_UNSET; - prgrp->mon.cntr_id[1] = MON_CNTR_UNSET; - list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, - mon.crdtgrp_list) { - crgrp->mon.cntr_id[0] = MON_CNTR_UNSET; - crgrp->mon.cntr_id[1] = MON_CNTR_UNSET; - } - } -} - -static ssize_t rdtgroup_mbm_mode_write(struct kernfs_open_file *of, - char *buf, size_t nbytes, - loff_t off) -{ - int mbm_cntr_assign = resctrl_arch_get_abmc_enabled(); - struct rdt_resource *r = of->kn->parent->priv; - int ret = 0; - - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - - buf[nbytes - 1] = '\0'; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - - rdt_last_cmd_clear(); - - if (!strcmp(buf, "legacy")) { - if (mbm_cntr_assign) - resctrl_arch_mbm_cntr_assign_disable(); - } else if (!strcmp(buf, "mbm_cntr_assign")) { - if (!mbm_cntr_assign) { - rdtgroup_mbm_cntr_reset(r); - ret = resctrl_arch_mbm_cntr_assign_enable(); - } - } else { - ret = -EINVAL; - } - - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); - - return ret ?: nbytes; -} - -static int rdtgroup_num_mbm_cntrs_show(struct kernfs_open_file *of, - struct seq_file *s, void *v) -{ - struct rdt_resource *r = of->kn->parent->priv; - - seq_printf(s, "%d\n", r->mon.num_mbm_cntrs); - - return 0; -} - -static char *rdtgroup_mon_state_to_str(struct rdtgroup *rdtgrp, - struct rdt_domain *d, char *str) -{ - char *tmp = str; - int index; - - /* - * Query the monitor state for the domain. - * Index 0 for evtid == QOS_L3_MBM_TOTAL_EVENT_ID - * Index 1 for evtid == QOS_L3_MBM_LOCAL_EVENT_ID - */ - index = mon_event_config_index_get(QOS_L3_MBM_TOTAL_EVENT_ID); - if (rdtgrp->mon.cntr_id[index] != MON_CNTR_UNSET && - test_bit(rdtgrp->mon.cntr_id[index], d->mbm_cntr_map)) - *tmp++ = 't'; - - index = mon_event_config_index_get(QOS_L3_MBM_LOCAL_EVENT_ID); - if (rdtgrp->mon.cntr_id[index] != MON_CNTR_UNSET && - test_bit(rdtgrp->mon.cntr_id[index], d->mbm_cntr_map)) - *tmp++ = 'l'; - - if (tmp == str) - *tmp++ = '_'; - - *tmp = '\0'; - return str; -} - -static int rdtgroup_mbm_control_show(struct kernfs_open_file *of, - struct seq_file *s, void *v) -{ - struct rdt_resource *r = of->kn->parent->priv; - struct rdt_domain *dom; - struct rdtgroup *rdtg; - char str[10]; - - if (!resctrl_arch_get_mbm_cntr_assign_enable()) { - rdt_last_cmd_puts("ABMC feature is not enabled\n"); - return -EINVAL; - } - - mutex_lock(&rdtgroup_mutex); - - list_for_each_entry(rdtg, &rdt_all_groups, rdtgroup_list) { - struct rdtgroup *crg; - - seq_printf(s, "%s//", rdtg->kn->name); - - list_for_each_entry(dom, &r->domains, list) - seq_printf(s, "%d=%s;", dom->id, - rdtgroup_mon_state_to_str(rdtg, dom, str)); - seq_putc(s, '\n'); - - list_for_each_entry(crg, &rdtg->mon.crdtgrp_list, - mon.crdtgrp_list) { - seq_printf(s, "%s/%s/", rdtg->kn->name, crg->kn->name); - - list_for_each_entry(dom, &r->domains, list) - seq_printf(s, "%d=%s;", dom->id, - rdtgroup_mon_state_to_str(crg, dom, str)); - seq_putc(s, '\n'); - } - } - - mutex_unlock(&rdtgroup_mutex); - return 0; -} - -/* - * Update the assign states for the domain. - * - * If this is a new assignment for the group then allocate a counter and update - * the assignment else just update the assign state - */ -static int rdtgroup_assign_update(struct rdtgroup *rdtgrp, enum resctrl_event_id evtid, - struct rdt_domain *d) -{ - int ret, index; - - index = mon_event_config_index_get(evtid); - if (index == INVALID_CONFIG_INDEX) - return -EINVAL; - - if (rdtgrp->mon.cntr_id[index] == MON_CNTR_UNSET) { - ret = rdtgroup_alloc_cntr(rdtgrp, index); - if (ret < 0) - goto out_done; - } - - /* Update the state on all domains if d == NULL */ - if (d == NULL) { - ret = rdtgroup_assign_cntr(rdtgrp, evtid); - } else { - ret = resctrl_arch_assign_cntr(d, evtid, rdtgrp->mon.rmid, - rdtgrp->mon.cntr_id[index], - rdtgrp->closid, 1); - if (!ret) - set_bit(rdtgrp->mon.cntr_id[index], d->mbm_cntr_map); - } - -out_done: - return ret; -} - -/* - * Update the unassign state for the domain. - * - * Free the counter if it is unassigned on all the domains else just - * update the unassign state - */ -static int rdtgroup_unassign_update(struct rdtgroup *rdtgrp, enum resctrl_event_id evtid, - struct rdt_domain *d) -{ - struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - int ret = 0, index; - - index = mon_event_config_index_get(evtid); - if (index == INVALID_CONFIG_INDEX) - return -EINVAL; - - if (rdtgrp->mon.cntr_id[index] == MON_CNTR_UNSET) - goto out_done; - - if (d == NULL) { - ret = rdtgroup_unassign_cntr(rdtgrp, evtid); - } else { - ret = resctrl_arch_assign_cntr(d, evtid, rdtgrp->mon.rmid, - rdtgrp->mon.cntr_id[index], - rdtgrp->closid, 0); - if (!ret) { - clear_bit(rdtgrp->mon.cntr_id[index], d->mbm_cntr_map); - rdtgroup_free_cntr(r, rdtgrp, index); - } - } - -out_done: - return ret; -} - -static int rdtgroup_str_to_mon_state(char *flag) -{ - int i, mon_state = 0; - - for (i = 0; i < strlen(flag); i++) { - switch (*(flag + i)) { - case 't': - mon_state |= ASSIGN_TOTAL; - break; - case 'l': - mon_state |= ASSIGN_LOCAL; - break; - case '_': - mon_state = ASSIGN_NONE; - break; - default: - break; - } - } - - return mon_state; -} - -static struct rdtgroup *rdtgroup_find_grp(enum rdt_group_type rtype, char *p_grp, char *c_grp) -{ - struct rdtgroup *rdtg, *crg; - - if (rtype == RDTCTRL_GROUP && *p_grp == '\0') { - return &rdtgroup_default; - } else if (rtype == RDTCTRL_GROUP) { - list_for_each_entry(rdtg, &rdt_all_groups, rdtgroup_list) - if (!strcmp(p_grp, rdtg->kn->name)) - return rdtg; - } else if (rtype == RDTMON_GROUP) { - list_for_each_entry(rdtg, &rdt_all_groups, rdtgroup_list) { - if (!strcmp(p_grp, rdtg->kn->name)) { - list_for_each_entry(crg, &rdtg->mon.crdtgrp_list, - mon.crdtgrp_list) { - if (!strcmp(c_grp, crg->kn->name)) - return crg; - } - } - } - } - - return NULL; -} - -static int rdtgroup_process_flags(struct rdt_resource *r, - enum rdt_group_type rtype, - char *p_grp, char *c_grp, char *tok) -{ - int op, mon_state, assign_state, unassign_state; - char *dom_str, *id_str, *op_str; - struct rdt_domain *d; - struct rdtgroup *rdtgrp; - unsigned long dom_id; - int ret, found = 0; - - rdtgrp = rdtgroup_find_grp(rtype, p_grp, c_grp); - - if (!rdtgrp) { - rdt_last_cmd_puts("Not a valid resctrl group\n"); - return -EINVAL; - } - -next: - if (!tok || tok[0] == '\0') - return 0; - - /* Start processing the strings for each domain */ - dom_str = strim(strsep(&tok, ";")); - - op_str = strpbrk(dom_str, "=+-"); - - if (op_str) { - op = *op_str; - } else { - rdt_last_cmd_puts("Missing operation =, +, -, _ character\n"); - return -EINVAL; - } - - id_str = strsep(&dom_str, "=+-"); - - /* Check for domain id '*' which means all domains */ - if (id_str && *id_str == '*') { - d = NULL; - goto check_state; - } else if (!id_str || kstrtoul(id_str, 10, &dom_id)) { - rdt_last_cmd_puts("Missing domain id\n"); - return -EINVAL; - } - - /* Verify if the dom_id is valid */ - list_for_each_entry(d, &r->domains, list) { - if (d->id == dom_id) { - found = 1; - break; - } - } - - if (!found) { - rdt_last_cmd_printf("Invalid domain id %ld\n", dom_id); - return -EINVAL; - } - -check_state: - mon_state = rdtgroup_str_to_mon_state(dom_str); - - assign_state = 0; - unassign_state = 0; - - switch (op) { - case '+': - if (mon_state == ASSIGN_NONE) { - rdt_last_cmd_puts("Invalid assign opcode\n"); - goto out_fail; - } - assign_state = mon_state; - break; - case '-': - if (mon_state == ASSIGN_NONE) { - rdt_last_cmd_puts("Invalid assign opcode\n"); - goto out_fail; - } - unassign_state = mon_state; - break; - case '=': - assign_state = mon_state; - unassign_state = (ASSIGN_TOTAL | ASSIGN_LOCAL) & ~assign_state; - break; - default: - break; - } - - if (assign_state & ASSIGN_TOTAL) { - ret = rdtgroup_assign_update(rdtgrp, QOS_L3_MBM_TOTAL_EVENT_ID, d); - if (ret) - goto out_fail; - } - - if (assign_state & ASSIGN_LOCAL) { - ret = rdtgroup_assign_update(rdtgrp, QOS_L3_MBM_LOCAL_EVENT_ID, d); - if (ret) - goto out_fail; - } - - if (unassign_state & ASSIGN_TOTAL) { - ret = rdtgroup_unassign_update(rdtgrp, QOS_L3_MBM_TOTAL_EVENT_ID, d); - if (ret) - goto out_fail; - } - - if (unassign_state & ASSIGN_LOCAL) { - ret = rdtgroup_unassign_update(rdtgrp, QOS_L3_MBM_LOCAL_EVENT_ID, d); - if (ret) - goto out_fail; - } - - goto next; - -out_fail: - - return -EINVAL; -} - -static ssize_t rdtgroup_mbm_control_write(struct kernfs_open_file *of, - char *buf, size_t nbytes, - loff_t off) -{ - struct rdt_resource *r = of->kn->parent->priv; - char *token, *cmon_grp, *mon_grp; - int ret; - - if (!resctrl_arch_get_abmc_enabled()) - return -EINVAL; - - /* Valid input requires a trailing newline */ - if (nbytes == 0 || buf[nbytes - 1] != '\n') - return -EINVAL; - - buf[nbytes - 1] = '\0'; - - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); - rdt_last_cmd_clear(); - - while ((token = strsep(&buf, "\n")) != NULL) { - if (strstr(token, "//")) { - /* - * The CTRL_MON group processing: - * default CTRL_MON group: "//" - * non-default CTRL_MON group: "//flags" - * The CTRL_MON group will be empty string if it is a - * default group. - */ - cmon_grp = strsep(&token, "//"); - - /* - * strsep returns empty string for contiguous delimiters. - * Make sure check for two consecutive delimiters and - * advance the token. - */ - mon_grp = strsep(&token, "//"); - if (*mon_grp != '\0') { - rdt_last_cmd_printf("Invalid CTRL_MON group format %s\n", token); - ret = -EINVAL; - break; - } - - ret = rdtgroup_process_flags(r, RDTCTRL_GROUP, cmon_grp, mon_grp, token); - if (ret) - break; - } else if (strstr(token, "/")) { - /* - * MON group processing: - * MON_GROUP inside default CTRL_MON group: "//" - * MON_GROUP within CTRL_MON group: "//" - */ - cmon_grp = strsep(&token, "/"); - - /* Extract the MON_GROUP. It cannot be empty string */ - mon_grp = strsep(&token, "/"); - if (*mon_grp == '\0') { - rdt_last_cmd_printf("Invalid MON_GROUP format %s\n", token); - ret = -EINVAL; - break; - } - - ret = rdtgroup_process_flags(r, RDTMON_GROUP, cmon_grp, mon_grp, token); - if (ret) - break; - } - } - - mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); - - return ret ?: nbytes; -} - #ifdef CONFIG_PROC_CPU_RESCTRL - /* * A task can only be part of one resctrl control group and of one monitor * group which is associated to that control group. @@ -1422,14 +941,14 @@ int proc_resctrl_show(struct seq_file *s, struct pid_namespace *ns, continue; seq_printf(s, "res:%s%s\n", (rdtg == &rdtgroup_default) ? "/" : "", - rdtg->kn->name); + rdt_kn_name(rdtg->kn)); seq_puts(s, "mon:"); list_for_each_entry(crg, &rdtg->mon.crdtgrp_list, mon.crdtgrp_list) { if (!resctrl_arch_match_rmid(tsk, crg->mon.parent->closid, crg->mon.rmid)) continue; - seq_printf(s, "%s", crg->kn->name); + seq_printf(s, "%s", rdt_kn_name(crg->kn)); break; } seq_putc(s, '\n'); @@ -1462,29 +981,39 @@ static int rdt_last_cmd_status_show(struct kernfs_open_file *of, return 0; } +static void *rdt_kn_parent_priv(struct kernfs_node *kn) +{ + /* + * The parent pointer is only valid within RCU section since it can be + * replaced. + */ + guard(rcu)(); + return rcu_dereference(kn->__parent)->priv; +} + static int rdt_num_closids_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); seq_printf(seq, "%u\n", s->num_closid); return 0; } static int rdt_default_ctrl_show(struct kernfs_open_file *of, - struct seq_file *seq, void *v) + struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; - seq_printf(seq, "%x\n", r->default_ctrl); + seq_printf(seq, "%x\n", resctrl_get_default_ctrl(r)); return 0; } static int rdt_min_cbm_bits_show(struct kernfs_open_file *of, - struct seq_file *seq, void *v) + struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; seq_printf(seq, "%u\n", r->cache.min_cbm_bits); @@ -1494,7 +1023,7 @@ static int rdt_min_cbm_bits_show(struct kernfs_open_file *of, static int rdt_shareable_bits_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; seq_printf(seq, "%x\n", r->cache.shareable_bits); @@ -1518,7 +1047,7 @@ static int rdt_shareable_bits_show(struct kernfs_open_file *of, static int rdt_bit_usage_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); /* * Use unsigned long even though only 32 bits are used to ensure * test_bit() is used safely. @@ -1526,7 +1055,7 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of, unsigned long sw_shareable = 0, hw_shareable = 0; unsigned long exclusive = 0, pseudo_locked = 0; struct rdt_resource *r = s->res; - struct rdt_domain *dom; + struct rdt_ctrl_domain *dom; int i, hwb, swb, excl, psl; enum rdtgrp_mode mode; bool sep = false; @@ -1535,12 +1064,12 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of, cpus_read_lock(); mutex_lock(&rdtgroup_mutex); hw_shareable = r->cache.shareable_bits; - list_for_each_entry(dom, &r->domains, list) { + list_for_each_entry(dom, &r->ctrl_domains, hdr.list) { if (sep) seq_putc(seq, ';'); sw_shareable = 0; exclusive = 0; - seq_printf(seq, "%d=", dom->id); + seq_printf(seq, "%d=", dom->hdr.id); for (i = 0; i < closids_supported(); i++) { if (!closid_allocated(i)) continue; @@ -1598,9 +1127,9 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of, } static int rdt_min_bw_show(struct kernfs_open_file *of, - struct seq_file *seq, void *v) + struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; seq_printf(seq, "%u\n", r->membw.min_bw); @@ -1610,9 +1139,9 @@ static int rdt_min_bw_show(struct kernfs_open_file *of, static int rdt_num_rmids_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct rdt_resource *r = of->kn->parent->priv; + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); - seq_printf(seq, "%d\n", r->mon.num_rmid); + seq_printf(seq, "%d\n", r->num_rmid); return 0; } @@ -1620,10 +1149,10 @@ static int rdt_num_rmids_show(struct kernfs_open_file *of, static int rdt_mon_features_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct rdt_resource *r = of->kn->parent->priv; + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); struct mon_evt *mevt; - list_for_each_entry(mevt, &r->mon.evt_list, list) { + list_for_each_entry(mevt, &r->evt_list, list) { seq_printf(seq, "%s\n", mevt->name); if (mevt->configurable) seq_printf(seq, "%s_config\n", mevt->name); @@ -1633,9 +1162,9 @@ static int rdt_mon_features_show(struct kernfs_open_file *of, } static int rdt_bw_gran_show(struct kernfs_open_file *of, - struct seq_file *seq, void *v) + struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; seq_printf(seq, "%u\n", r->membw.bw_gran); @@ -1643,9 +1172,9 @@ static int rdt_bw_gran_show(struct kernfs_open_file *of, } static int rdt_delay_linear_show(struct kernfs_open_file *of, - struct seq_file *seq, void *v) + struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; seq_printf(seq, "%u\n", r->membw.delay_linear); @@ -1663,13 +1192,22 @@ static int max_threshold_occ_show(struct kernfs_open_file *of, static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; - if (r->membw.throttle_mode == THREAD_THROTTLE_PER_THREAD) + switch (r->membw.throttle_mode) { + case THREAD_THROTTLE_PER_THREAD: seq_puts(seq, "per-thread\n"); - else + return 0; + case THREAD_THROTTLE_MAX: seq_puts(seq, "max\n"); + return 0; + case THREAD_THROTTLE_UNDEFINED: + seq_puts(seq, "undefined\n"); + return 0; + } + + WARN_ON_ONCE(1); return 0; } @@ -1712,7 +1250,7 @@ static int rdtgroup_mode_show(struct kernfs_open_file *of, return 0; } -enum resctrl_conf_type resctrl_peer_type(enum resctrl_conf_type my_type) +static enum resctrl_conf_type resctrl_peer_type(enum resctrl_conf_type my_type) { switch (my_type) { case CDP_CODE: @@ -1728,7 +1266,7 @@ enum resctrl_conf_type resctrl_peer_type(enum resctrl_conf_type my_type) static int rdt_has_sparse_bitmasks_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct resctrl_schema *s = of->kn->parent->priv; + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; seq_printf(seq, "%u\n", r->cache.arch_has_sparse_bitmasks); @@ -1757,7 +1295,7 @@ static int rdt_has_sparse_bitmasks_show(struct kernfs_open_file *of, * * Return: false if CBM does not overlap, true if it does. */ -static bool __rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d, +static bool __rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_ctrl_domain *d, unsigned long cbm, int closid, enum resctrl_conf_type type, bool exclusive) { @@ -1812,7 +1350,7 @@ static bool __rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d * * Return: true if CBM overlap detected, false if there is no overlap */ -bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_domain *d, +bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_ctrl_domain *d, unsigned long cbm, int closid, bool exclusive) { enum resctrl_conf_type peer_type = resctrl_peer_type(s->conf_type); @@ -1843,10 +1381,10 @@ bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_domain *d, static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp) { int closid = rdtgrp->closid; + struct rdt_ctrl_domain *d; struct resctrl_schema *s; struct rdt_resource *r; bool has_cache = false; - struct rdt_domain *d; u32 ctrl; /* Walking r->domains, ensure it can't race with cpuhp */ @@ -1857,7 +1395,7 @@ static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp) if (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA) continue; has_cache = true; - list_for_each_entry(d, &r->domains, list) { + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { ctrl = resctrl_arch_get_config(r, d, closid, s->conf_type); if (rdtgroup_cbm_overlaps(s, d, ctrl, closid, false)) { @@ -1963,24 +1501,38 @@ static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of, * bitmap functions work correctly. */ unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, - struct rdt_domain *d, unsigned long cbm) + struct rdt_ctrl_domain *d, unsigned long cbm) { - struct cpu_cacheinfo *ci; unsigned int size = 0; - int num_b, i; + struct cacheinfo *ci; + int num_b; + + if (WARN_ON_ONCE(r->ctrl_scope != RESCTRL_L2_CACHE && r->ctrl_scope != RESCTRL_L3_CACHE)) + return size; num_b = bitmap_weight(&cbm, r->cache.cbm_len); - ci = get_cpu_cacheinfo(cpumask_any(&d->cpu_mask)); - for (i = 0; i < ci->num_leaves; i++) { - if (ci->info_list[i].level == r->cache_level) { - size = ci->info_list[i].size / r->cache.cbm_len * num_b; - break; - } - } + ci = get_cpu_cacheinfo_level(cpumask_any(&d->hdr.cpu_mask), r->ctrl_scope); + if (ci) + size = ci->size / r->cache.cbm_len * num_b; return size; } +bool is_mba_sc(struct rdt_resource *r) +{ + if (!r) + r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + + /* + * The software controller support is only applicable to MBA resource. + * Make sure to check for resource type. + */ + if (r->rid != RDT_RESOURCE_MBA) + return false; + + return r->membw.mba_sc; +} + /* * rdtgroup_size_show - Display size in bytes of allocated regions * @@ -1992,9 +1544,9 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, { struct resctrl_schema *schema; enum resctrl_conf_type type; + struct rdt_ctrl_domain *d; struct rdtgroup *rdtgrp; struct rdt_resource *r; - struct rdt_domain *d; unsigned int size; int ret = 0; u32 closid; @@ -2018,7 +1570,7 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, size = rdtgroup_cbm_to_size(rdtgrp->plr->s->res, rdtgrp->plr->d, rdtgrp->plr->cbm); - seq_printf(s, "%d=%u\n", rdtgrp->plr->d->id, size); + seq_printf(s, "%d=%u\n", rdtgrp->plr->d->hdr.id, size); } goto out; } @@ -2030,7 +1582,7 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, type = schema->conf_type; sep = false; seq_printf(s, "%*s:", max_name_width, schema->name); - list_for_each_entry(d, &r->domains, list) { + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { if (sep) seq_putc(s, ';'); if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { @@ -2048,7 +1600,7 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, else size = rdtgroup_cbm_to_size(r, d, ctrl); } - seq_printf(s, "%d=%u", d->id, size); + seq_printf(s, "%d=%u", d->hdr.id, size); sep = true; } seq_putc(s, '\n'); @@ -2060,24 +1612,32 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, return ret; } +static void mondata_config_read(struct resctrl_mon_config_info *mon_info) +{ + smp_call_function_any(&mon_info->d->hdr.cpu_mask, + resctrl_arch_mon_event_config_read, mon_info, 1); +} + static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid) { - struct rdt_domain *dom; + struct resctrl_mon_config_info mon_info; + struct rdt_mon_domain *dom; bool sep = false; - u32 val; cpus_read_lock(); mutex_lock(&rdtgroup_mutex); - list_for_each_entry(dom, &r->domains, list) { + list_for_each_entry(dom, &r->mon_domains, hdr.list) { if (sep) seq_puts(s, ";"); - val = resctrl_arch_event_config_get(dom, evtid); - if (val == INVALID_CONFIG_VALUE) - break; + memset(&mon_info, 0, sizeof(struct resctrl_mon_config_info)); + mon_info.r = r; + mon_info.d = dom; + mon_info.evtid = evtid; + mondata_config_read(&mon_info); - seq_printf(s, "%d=0x%02x", dom->id, val); + seq_printf(s, "%d=0x%02x", dom->hdr.id, mon_info.mon_config); sep = true; } seq_puts(s, "\n"); @@ -2091,7 +1651,7 @@ static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid static int mbm_total_bytes_config_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct rdt_resource *r = of->kn->parent->priv; + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); mbm_config_show(seq, r, QOS_L3_MBM_TOTAL_EVENT_ID); @@ -2101,29 +1661,29 @@ static int mbm_total_bytes_config_show(struct kernfs_open_file *of, static int mbm_local_bytes_config_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct rdt_resource *r = of->kn->parent->priv; + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); mbm_config_show(seq, r, QOS_L3_MBM_LOCAL_EVENT_ID); return 0; } -static int mbm_config_write_domain(struct rdt_resource *r, - struct rdt_domain *d, u32 evtid, u32 val) +static void mbm_config_write_domain(struct rdt_resource *r, + struct rdt_mon_domain *d, u32 evtid, u32 val) { struct resctrl_mon_config_info mon_info = {0}; - u32 config_val; /* - * Check the current config value first. If both are the same then + * Read the current config value first. If both are the same then * no need to write it again. */ - config_val = resctrl_arch_event_config_get(d, evtid); - if (config_val == INVALID_CONFIG_VALUE || config_val == val) - return 0; - + mon_info.r = r; mon_info.d = d; mon_info.evtid = evtid; + mondata_config_read(&mon_info); + if (mon_info.mon_config == val) + return; + mon_info.mon_config = val; /* @@ -2132,12 +1692,8 @@ static int mbm_config_write_domain(struct rdt_resource *r, * are scoped at the domain level. Writing any of these MSRs * on one CPU is observed by all the CPUs in the domain. */ - smp_call_function_any(&d->cpu_mask, resctrl_arch_event_config_set, + smp_call_function_any(&d->hdr.cpu_mask, resctrl_arch_mon_event_config_write, &mon_info, 1); - if (mon_info.err) { - rdt_last_cmd_puts("Invalid event configuration\n"); - return mon_info.err; - } /* * When an Event Configuration is changed, the bandwidth counters @@ -2149,16 +1705,13 @@ static int mbm_config_write_domain(struct rdt_resource *r, * mbm_local and mbm_total counts for all the RMIDs. */ resctrl_arch_reset_rmid_all(r, d); - - return 0; } static int mon_config_write(struct rdt_resource *r, char *tok, u32 evtid) { char *dom_str = NULL, *id_str; unsigned long dom_id, val; - struct rdt_domain *d; - int err; + struct rdt_mon_domain *d; /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); @@ -2188,11 +1741,9 @@ static int mon_config_write(struct rdt_resource *r, char *tok, u32 evtid) return -EINVAL; } - list_for_each_entry(d, &r->domains, list) { - if (d->id == dom_id) { - err = mbm_config_write_domain(r, d, evtid, val); - if (err) - return err; + list_for_each_entry(d, &r->mon_domains, hdr.list) { + if (d->hdr.id == dom_id) { + mbm_config_write_domain(r, d, evtid, val); goto next; } } @@ -2204,7 +1755,7 @@ static ssize_t mbm_total_bytes_config_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - struct rdt_resource *r = of->kn->parent->priv; + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); int ret; /* Valid input requires a trailing newline */ @@ -2230,7 +1781,7 @@ static ssize_t mbm_local_bytes_config_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - struct rdt_resource *r = of->kn->parent->priv; + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); int ret; /* Valid input requires a trailing newline */ @@ -2252,124 +1803,6 @@ static ssize_t mbm_local_bytes_config_write(struct kernfs_open_file *of, return ret ?: nbytes; } -/* Allocate a new counter id if the event is unassigned */ -int rdtgroup_alloc_cntr(struct rdtgroup *rdtgrp, int index) -{ - struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - int cntr_id; - - /* Nothing to do if event has been assigned already */ - if (rdtgrp->mon.cntr_id[index] != MON_CNTR_UNSET) { - rdt_last_cmd_puts("ABMC counter is assigned already\n"); - return 0; - } - - /* - * Allocate a new counter id and update domains - */ - cntr_id = mbm_cntr_alloc(r); - if (cntr_id < 0) { - rdt_last_cmd_puts("Out of ABMC counters\n"); - return -ENOSPC; - } - - rdtgrp->mon.cntr_id[index] = cntr_id; - - return 0; -} - -/* - * Assign a hardware counter to the group and assign the counter - * all the domains in the group. It will try to allocate the mbm - * counter if the counter is available. - */ -int rdtgroup_assign_cntr(struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) -{ - struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - struct rdt_domain *d; - int index; - - index = mon_event_config_index_get(evtid); - if (index == INVALID_CONFIG_INDEX) - return -EINVAL; - - if (rdtgroup_alloc_cntr(rdtgrp, index)) - return -EINVAL; - - list_for_each_entry(d, &r->domains, list) { - resctrl_arch_assign_cntr(d, evtid, rdtgrp->mon.rmid, - rdtgrp->mon.cntr_id[index], - rdtgrp->closid, true); - set_bit(rdtgrp->mon.cntr_id[index], d->mbm_cntr_map); - } - - return 0; -} - -static int rdtgroup_mbm_cntr_test(struct rdt_resource *r, u32 cntr_id) -{ - struct rdt_domain *d; - - list_for_each_entry(d, &r->domains, list) - if (test_bit(cntr_id, d->mbm_cntr_map)) - return 1; - - return 0; -} - -/* Free the counter id after the event is unassigned */ -void rdtgroup_free_cntr(struct rdt_resource *r, struct rdtgroup *rdtgrp, - int index) -{ - /* Update the counter bitmap */ - if (!rdtgroup_mbm_cntr_test(r, rdtgrp->mon.cntr_id[index])) { - mbm_cntr_free(rdtgrp->mon.cntr_id[index]); - rdtgrp->mon.cntr_id[index] = MON_CNTR_UNSET; - } -} - -/* - * Unassign a hardware counter from the group and update all the domains - * in the group. - */ -int rdtgroup_unassign_cntr(struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) -{ - struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - struct rdt_domain *d; - int index; - - index = mon_event_config_index_get(evtid); - if (index == INVALID_CONFIG_INDEX) - return -EINVAL; - - if (rdtgrp->mon.cntr_id[index] != MON_CNTR_UNSET) { - list_for_each_entry(d, &r->domains, list) { - resctrl_arch_assign_cntr(d, evtid, rdtgrp->mon.rmid, - rdtgrp->mon.cntr_id[index], - rdtgrp->closid, false); - clear_bit(rdtgrp->mon.cntr_id[index], - d->mbm_cntr_map); - } - - /* Free the counter at group level */ - rdtgroup_free_cntr(r, rdtgrp, index); - } - - return 0; -} - -const char *rdtgroup_name_by_closid(u32 closid) -{ - struct rdtgroup *rdtgrp; - - list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { - if (rdtgrp->closid == closid) - return rdtgrp->kn->name; - } - - return NULL; -} - /* rdtgroup information files for one cache resource. */ static struct rftype res_common_files[] = { { @@ -2460,22 +1893,6 @@ static struct rftype res_common_files[] = { .kf_ops = &rdtgroup_kf_single_ops, .seq_show = rdt_thread_throttle_mode_show, }, -#ifdef CONFIG_X86 - { - .name = "io_alloc", - .mode = 0644, - .kf_ops = &rdtgroup_kf_single_ops, - .seq_show = resctrl_io_alloc_show, - .write = resctrl_io_alloc_write, - }, - { - .name = "io_alloc_cbm", - .mode = 0644, - .kf_ops = &rdtgroup_kf_single_ops, - .seq_show = resctrl_io_alloc_cbm_show, - .write = resctrl_io_alloc_cbm_write, - }, -#endif { .name = "max_threshold_occupancy", .mode = 0644, @@ -2498,14 +1915,6 @@ static struct rftype res_common_files[] = { .seq_show = mbm_local_bytes_config_show, .write = mbm_local_bytes_config_write, }, - { - .name = "mbm_mode", - .mode = 0644, - .kf_ops = &rdtgroup_kf_single_ops, - .seq_show = rdtgroup_mbm_mode_show, - .write = rdtgroup_mbm_mode_write, - .fflags = RFTYPE_MON_INFO, - }, { .name = "cpus", .mode = 0644, @@ -2514,19 +1923,6 @@ static struct rftype res_common_files[] = { .seq_show = rdtgroup_cpus_show, .fflags = RFTYPE_BASE, }, - { - .name = "num_mbm_cntrs", - .mode = 0444, - .kf_ops = &rdtgroup_kf_single_ops, - .seq_show = rdtgroup_num_mbm_cntrs_show, - }, - { - .name = "mbm_control", - .mode = 0644, - .kf_ops = &rdtgroup_kf_single_ops, - .seq_show = rdtgroup_mbm_control_show, - .write = rdtgroup_mbm_control_write, - }, { .name = "cpus_list", .mode = 0644, @@ -2559,6 +1955,13 @@ static struct rftype res_common_files[] = { .seq_show = rdtgroup_schemata_show, .fflags = RFTYPE_CTRL_BASE, }, + { + .name = "mba_MBps_event", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_mba_mbps_event_write, + .seq_show = rdtgroup_mba_mbps_event_show, + }, { .name = "mode", .mode = 0644, @@ -2588,7 +1991,6 @@ static struct rftype res_common_files[] = { .seq_show = rdtgroup_closid_show, .fflags = RFTYPE_CTRL_BASE | RFTYPE_DEBUG, }, - }; static int rdtgroup_add_files(struct kernfs_node *kn, unsigned long fflags) @@ -2638,24 +2040,27 @@ static struct rftype *rdtgroup_get_rftype_by_name(const char *name) return NULL; } -#ifdef CONFIG_X86 -/* - * The resctrl file "io_alloc" is added using L3 resource. However, it results - * in this file being visible for *all* cache resources (eg. L2 cache), - * whether it supports "io_alloc" or not. - */ -static void io_alloc_init(void) +static void thread_throttle_mode_init(void) { - struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + enum membw_throttle_mode throttle_mode = THREAD_THROTTLE_UNDEFINED; + struct rdt_resource *r_mba, *r_smba; - if (r->cache.io_alloc_capable) { - resctrl_file_fflags_init("io_alloc", RFTYPE_CTRL_INFO | - RFTYPE_RES_CACHE); - resctrl_file_fflags_init("io_alloc_cbm", - RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE); - } + r_mba = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + if (r_mba->alloc_capable && + r_mba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_mba->membw.throttle_mode; + + r_smba = resctrl_arch_get_resource(RDT_RESOURCE_SMBA); + if (r_smba->alloc_capable && + r_smba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_smba->membw.throttle_mode; + + if (throttle_mode == THREAD_THROTTLE_UNDEFINED) + return; + + resctrl_file_fflags_init("thread_throttle_mode", + RFTYPE_CTRL_INFO | RFTYPE_RES_MB); } -#endif void resctrl_file_fflags_init(const char *config, unsigned long fflags) { @@ -2785,9 +2190,22 @@ static int rdtgroup_mkdir_info_resdir(void *priv, char *name, return ret; } +static unsigned long fflags_from_resource(struct rdt_resource *r) +{ + switch (r->rid) { + case RDT_RESOURCE_L3: + case RDT_RESOURCE_L2: + return RFTYPE_RES_CACHE; + case RDT_RESOURCE_MBA: + case RDT_RESOURCE_SMBA: + return RFTYPE_RES_MB; + } + + return WARN_ON_ONCE(1); +} + static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) { - enum resctrl_res_level i; struct resctrl_schema *s; struct rdt_resource *r; unsigned long fflags; @@ -2806,18 +2224,14 @@ static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) /* loop over enabled controls, these are all alloc_capable */ list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; - fflags = r->fflags | RFTYPE_CTRL_INFO; + fflags = fflags_from_resource(r) | RFTYPE_CTRL_INFO; ret = rdtgroup_mkdir_info_resdir(s, s->name, fflags); if (ret) goto out_destroy; } - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - r = resctrl_arch_get_resource(i); - if (!r->mon_capable) - continue; - - fflags = r->fflags | RFTYPE_MON_INFO; + for_each_mon_capable_rdt_resource(r) { + fflags = fflags_from_resource(r) | RFTYPE_MON_INFO; sprintf(name, "%s_MON", r->name); ret = rdtgroup_mkdir_info_resdir(r, name, fflags); if (ret) @@ -2870,10 +2284,10 @@ static inline bool is_mba_linear(void) return resctrl_arch_get_resource(RDT_RESOURCE_MBA)->membw.delay_linear; } -static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_domain *d) +static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_ctrl_domain *d) { u32 num_closid = resctrl_arch_get_num_closid(r); - int cpu = cpumask_any(&d->cpu_mask); + int cpu = cpumask_any(&d->hdr.cpu_mask); int i; d->mbps_val = kcalloc_node(num_closid, sizeof(*d->mbps_val), @@ -2888,7 +2302,7 @@ static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_domain *d) } static void mba_sc_domain_destroy(struct rdt_resource *r, - struct rdt_domain *d) + struct rdt_ctrl_domain *d) { kfree(d->mbps_val); d->mbps_val = NULL; @@ -2896,14 +2310,18 @@ static void mba_sc_domain_destroy(struct rdt_resource *r, /* * MBA software controller is supported only if - * MBM is supported and MBA is in linear scale. + * MBM is supported and MBA is in linear scale, + * and the MBM monitor scope is the same as MBA + * control scope. */ static bool supports_mba_mbps(void) { + struct rdt_resource *rmbm = resctrl_arch_get_resource(RDT_RESOURCE_L3); struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); - return (resctrl_arch_is_mbm_local_enabled() && - r->alloc_capable && is_mba_linear()); + return (resctrl_is_mbm_enabled() && + r->alloc_capable && is_mba_linear() && + r->ctrl_scope == rmbm->mon_scope); } /* @@ -2914,7 +2332,8 @@ static int set_mba_sc(bool mba_sc) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); u32 num_closid = resctrl_arch_get_num_closid(r); - struct rdt_domain *d; + struct rdt_ctrl_domain *d; + unsigned long fflags; int i; if (!supports_mba_mbps() || mba_sc == is_mba_sc(r)) @@ -2922,11 +2341,16 @@ static int set_mba_sc(bool mba_sc) r->membw.mba_sc = mba_sc; - list_for_each_entry(d, &r->domains, list) { + rdtgroup_default.mba_mbps_event = mba_mbps_default_event; + + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { for (i = 0; i < num_closid; i++) d->mbps_val[i] = MBA_MAX_MBPS; } + fflags = mba_sc ? RFTYPE_CTRL_BASE | RFTYPE_MON_BASE : 0; + resctrl_file_fflags_init("mba_MBps_event", fflags); + return 0; } @@ -2947,12 +2371,13 @@ static struct rdtgroup *kernfs_to_rdtgroup(struct kernfs_node *kn) * resource. "info" and its subdirectories don't * have rdtgroup structures, so return NULL here. */ - if (kn == kn_info || kn->parent == kn_info) + if (kn == kn_info || + rcu_access_pointer(kn->__parent) == kn_info) return NULL; else return kn->priv; } else { - return kn->parent->priv; + return rdt_kn_parent_priv(kn); } } @@ -3017,7 +2442,6 @@ static void rdt_disable_ctx(void) resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L3, false); resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L2, false); set_mba_sc(false); - resctrl_arch_set_hwdrc_enabled(RDT_RESOURCE_MBA, false); resctrl_debug = false; } @@ -3038,19 +2462,6 @@ static int rdt_enable_ctx(struct rdt_fs_context *ctx) goto out_cdpl2; } - /* - * MBA and memory bandwidth HWDRC features are mutually exclusive. - * So mba_MBps and hwdrc_mb options could not be set at the same time. - */ - if (ctx->enable_hwdrc_mb && ctx->enable_mba_mbps) { - pr_debug("Option 'mba_MBps' and 'hwdrc_mb' are mutually exclusive\n"); - ret = -EINVAL; - goto out_cdpl3; - } - - if (!ret && ctx->enable_hwdrc_mb) - ret = resctrl_arch_set_hwdrc_enabled(RDT_RESOURCE_MBA, true); - if (ctx->enable_mba_mbps) { ret = set_mba_sc(true); if (ret) @@ -3117,11 +2528,19 @@ static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type if (cl > max_name_width) max_name_width = cl; - /* - * Choose a width for the resource data based on the resource that has - * widest name and cbm. - */ - max_data_width = max(max_data_width, r->data_width); + switch (r->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + s->fmt_str = "%d=%x"; + break; + case RESCTRL_SCHEMA_RANGE: + s->fmt_str = "%d=%u"; + break; + } + + if (WARN_ON_ONCE(!s->fmt_str)) { + kfree(s); + return -EINVAL; + } INIT_LIST_HEAD(&s->list); list_add(&s->list, &resctrl_schema_all); @@ -3131,15 +2550,10 @@ static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type static int schemata_list_create(void) { - enum resctrl_res_level i; struct rdt_resource *r; int ret = 0; - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - r = resctrl_arch_get_resource(i); - if (!r->alloc_capable) - continue; - + for_each_alloc_capable_rdt_resource(r) { if (resctrl_arch_get_cdp_enabled(r->rid)) { ret = schemata_list_add(r, CDP_CODE); if (ret) @@ -3167,52 +2581,12 @@ static void schemata_list_destroy(void) } } -/* - * Called when new group is created. Assign the counters if ABMC is - * already enabled. Two counters are required per group, one for total - * event and one for local event. With limited number of counters, - * the assignments can fail in some cases. But, it is not required to - * fail the group creation. Users have the option to modify the - * assignments after the group creation. - */ -static int rdtgroup_assign_cntrs(struct rdtgroup *rdtgrp) -{ - int ret = 0; - - if (!resctrl_arch_get_abmc_enabled()) - return 0; - - if (resctrl_arch_is_mbm_total_enabled()) - ret = rdtgroup_assign_cntr(rdtgrp, QOS_L3_MBM_TOTAL_EVENT_ID); - - if (!ret && resctrl_arch_is_mbm_local_enabled()) - ret = rdtgroup_assign_cntr(rdtgrp, QOS_L3_MBM_LOCAL_EVENT_ID); - - return ret; -} - -static int rdtgroup_unassign_cntrs(struct rdtgroup *rdtgrp) -{ - int ret = 0; - - if (!resctrl_arch_get_abmc_enabled()) - return 0; - - if (resctrl_arch_is_mbm_total_enabled()) - ret = rdtgroup_unassign_cntr(rdtgrp, QOS_L3_MBM_TOTAL_EVENT_ID); - - if (!ret && resctrl_arch_is_mbm_local_enabled()) - ret = rdtgroup_unassign_cntr(rdtgrp, QOS_L3_MBM_LOCAL_EVENT_ID); - - return ret; -} - static int rdt_get_tree(struct fs_context *fc) { - struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); struct rdt_fs_context *ctx = rdt_fc2context(fc); unsigned long flags = RFTYPE_CTRL_BASE; - struct rdt_domain *dom; + struct rdt_mon_domain *dom; + struct rdt_resource *r; int ret; cpus_read_lock(); @@ -3234,27 +2608,25 @@ static int rdt_get_tree(struct fs_context *fc) goto out_root; ret = schemata_list_create(); - if (ret) { - schemata_list_destroy(); - goto out_ctx; - } - - closid_init(); + if (ret) + goto out_schemata_free; - mbm_cntrs_init(resctrl_arch_get_resource(RDT_RESOURCE_L3)); + ret = closid_init(); + if (ret) + goto out_schemata_free; if (resctrl_arch_mon_capable()) flags |= RFTYPE_MON; ret = rdtgroup_add_files(rdtgroup_default.kn, flags); if (ret) - goto out_schemata_free; + goto out_closid_exit; kernfs_activate(rdtgroup_default.kn); ret = rdtgroup_create_info_dir(rdtgroup_default.kn); if (ret < 0) - goto out_schemata_free; + goto out_closid_exit; if (resctrl_arch_mon_capable()) { ret = mongroup_create_dir(rdtgroup_default.kn, @@ -3268,8 +2640,6 @@ static int rdt_get_tree(struct fs_context *fc) if (ret < 0) goto out_mongrp; rdtgroup_default.mon.mon_data_kn = kn_mondata; - - rdtgroup_assign_cntrs(&rdtgroup_default); } ret = rdt_pseudo_lock_init(); @@ -3289,7 +2659,8 @@ static int rdt_get_tree(struct fs_context *fc) resctrl_mounted = true; if (resctrl_is_mbm_enabled()) { - list_for_each_entry(dom, &l3->domains, list) + r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + list_for_each_entry(dom, &r->mon_domains, hdr.list) mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL, RESCTRL_PICK_ANY_CPU); } @@ -3299,7 +2670,6 @@ static int rdt_get_tree(struct fs_context *fc) out_psl: rdt_pseudo_lock_release(); out_mondata: - rdtgroup_unassign_cntrs(&rdtgroup_default); if (resctrl_arch_mon_capable()) kernfs_remove(kn_mondata); out_mongrp: @@ -3307,9 +2677,10 @@ static int rdt_get_tree(struct fs_context *fc) kernfs_remove(kn_mongrp); out_info: kernfs_remove(kn_info); +out_closid_exit: + closid_exit(); out_schemata_free: schemata_list_destroy(); -out_ctx: rdt_disable_ctx(); out_root: rdtgroup_destroy_root(); @@ -3324,7 +2695,6 @@ enum rdt_param { Opt_cdp, Opt_cdpl2, Opt_mba_mbps, - Opt_hwdrc_mb, Opt_debug, nr__rdt_params }; @@ -3333,7 +2703,6 @@ static const struct fs_parameter_spec rdt_fs_parameters[] = { fsparam_flag("cdp", Opt_cdp), fsparam_flag("cdpl2", Opt_cdpl2), fsparam_flag("mba_MBps", Opt_mba_mbps), - fsparam_flag("hwdrc_mb", Opt_hwdrc_mb), fsparam_flag("debug", Opt_debug), {} }; @@ -3342,6 +2711,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) { struct rdt_fs_context *ctx = rdt_fc2context(fc); struct fs_parse_result result; + const char *msg; int opt; opt = fs_parse(fc, rdt_fs_parameters, param, &result); @@ -3356,15 +2726,11 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) ctx->enable_cdpl2 = true; return 0; case Opt_mba_mbps: + msg = "mba_MBps requires MBM and linear scale MBA at L3 scope"; if (!supports_mba_mbps()) - return -EINVAL; + return invalfc(fc, msg); ctx->enable_mba_mbps = true; return 0; - case Opt_hwdrc_mb: - if (!resctrl_arch_is_hwdrc_mb_capable()) - return -EINVAL; - ctx->enable_hwdrc_mb = true; - return 0; case Opt_debug: ctx->enable_debug = true; return 0; @@ -3391,7 +2757,7 @@ static int rdt_init_fs_context(struct fs_context *fc) { struct rdt_fs_context *ctx; - ctx = kzalloc(sizeof(struct rdt_fs_context), GFP_KERNEL); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; @@ -3511,44 +2877,98 @@ static void rmdir_all_sub(void) kernfs_remove(kn_mondata); } -static void rdt_kill_sb(struct super_block *sb) +/** + * mon_get_kn_priv() - Get the mon_data priv data for this event. + * + * The same values are used across the mon_data directories of all control and + * monitor groups for the same event in the same domain. Keep a list of + * allocated structures and re-use an existing one with the same values for + * @rid, @domid, etc. + * + * @rid: The resource id for the event file being created. + * @domid: The domain id for the event file being created. + * @mevt: The type of event file being created. + * @do_sum: Whether SNC summing monitors are being created. + */ +static struct mon_data *mon_get_kn_priv(enum resctrl_res_level rid, int domid, + struct mon_evt *mevt, + bool do_sum) { - struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); - struct rdt_domain *d; + struct mon_data *priv; - cpus_read_lock(); - mutex_lock(&rdtgroup_mutex); + lockdep_assert_held(&rdtgroup_mutex); - rdt_disable_ctx(); + list_for_each_entry(priv, &mon_data_kn_priv_list, list) { + if (priv->rid == rid && priv->domid == domid && + priv->sum == do_sum && priv->evtid == mevt->evtid) + return priv; + } - /* Put everything back to default values. */ - resctrl_arch_reset_resources(); + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; - rmdir_all_sub(); + priv->rid = rid; + priv->domid = domid; + priv->sum = do_sum; + priv->evtid = mevt->evtid; + list_add_tail(&priv->list, &mon_data_kn_priv_list); - /* - * When resctrl is umounted, forcefully cancel delayed works since the - * new mount option may be changed. - */ - list_for_each_entry(d, &l3->domains, list) { - if (resctrl_is_mbm_enabled()) - cancel_delayed_work(&d->mbm_over); - if (resctrl_arch_is_llc_occupancy_enabled() && has_busy_rmid(d)) { - __check_limbo(d, true); - cancel_delayed_work(&d->cqm_limbo); - } + return priv; +} + +/** + * mon_put_kn_priv() - Free all allocated mon_data structures. + * + * Called when resctrl file system is unmounted. + */ +static void mon_put_kn_priv(void) +{ + struct mon_data *priv, *tmp; + + lockdep_assert_held(&rdtgroup_mutex); + + list_for_each_entry_safe(priv, tmp, &mon_data_kn_priv_list, list) { + list_del(&priv->list); + kfree(priv); } +} + +static void resctrl_fs_teardown(void) +{ + lockdep_assert_held(&rdtgroup_mutex); + + /* Cleared by rdtgroup_destroy_root() */ + if (!rdtgroup_default.kn) + return; + rmdir_all_sub(); + mon_put_kn_priv(); rdt_pseudo_lock_release(); rdtgroup_default.mode = RDT_MODE_SHAREABLE; + closid_exit(); schemata_list_destroy(); rdtgroup_destroy_root(); +} + +static void rdt_kill_sb(struct super_block *sb) +{ + struct rdt_resource *r; + + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + + rdt_disable_ctx(); + + /* Put everything back to default values. */ + for_each_alloc_capable_rdt_resource(r) + resctrl_arch_reset_all_ctrls(r); + + resctrl_fs_teardown(); if (resctrl_arch_alloc_capable()) resctrl_arch_disable_alloc(); if (resctrl_arch_mon_capable()) resctrl_arch_disable_mon(); - - rdtgroup_unassign_cntrs(&rdtgroup_default); resctrl_mounted = false; kernfs_kill_sb(sb); mutex_unlock(&rdtgroup_mutex); @@ -3583,62 +3003,127 @@ static int mon_addfile(struct kernfs_node *parent_kn, const char *name, return ret; } +static void mon_rmdir_one_subdir(struct kernfs_node *pkn, char *name, char *subname) +{ + struct kernfs_node *kn; + + kn = kernfs_find_and_get(pkn, name); + if (!kn) + return; + kernfs_put(kn); + + if (kn->dir.subdirs <= 1) + kernfs_remove(kn); + else + kernfs_remove_by_name(kn, subname); +} + /* * Remove all subdirectories of mon_data of ctrl_mon groups - * and monitor groups with given domain id. + * and monitor groups for the given domain. + * Remove files and directories containing "sum" of domain data + * when last domain being summed is removed. */ static void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, - unsigned int dom_id) + struct rdt_mon_domain *d) { struct rdtgroup *prgrp, *crgrp; + char subname[32]; + bool snc_mode; char name[32]; + snc_mode = r->mon_scope == RESCTRL_L3_NODE; + sprintf(name, "mon_%s_%02d", r->name, snc_mode ? d->ci_id : d->hdr.id); + if (snc_mode) + sprintf(subname, "mon_sub_%s_%02d", r->name, d->hdr.id); + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { - sprintf(name, "mon_%s_%02d", r->name, dom_id); - kernfs_remove_by_name(prgrp->mon.mon_data_kn, name); + mon_rmdir_one_subdir(prgrp->mon.mon_data_kn, name, subname); list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, mon.crdtgrp_list) - kernfs_remove_by_name(crgrp->mon.mon_data_kn, name); + mon_rmdir_one_subdir(crgrp->mon.mon_data_kn, name, subname); + } +} + +static int mon_add_all_files(struct kernfs_node *kn, struct rdt_mon_domain *d, + struct rdt_resource *r, struct rdtgroup *prgrp, + bool do_sum) +{ + struct rmid_read rr = {0}; + struct mon_data *priv; + struct mon_evt *mevt; + int ret, domid; + + if (WARN_ON(list_empty(&r->evt_list))) + return -EPERM; + + list_for_each_entry(mevt, &r->evt_list, list) { + domid = do_sum ? d->ci_id : d->hdr.id; + priv = mon_get_kn_priv(r->rid, domid, mevt, do_sum); + if (WARN_ON_ONCE(!priv)) + return -EINVAL; + + ret = mon_addfile(kn, mevt->name, priv); + if (ret) + return ret; + + if (!do_sum && resctrl_is_mbm_event(mevt->evtid)) + mon_event_read(&rr, r, d, prgrp, &d->hdr.cpu_mask, mevt->evtid, true); } + + return 0; } static int mkdir_mondata_subdir(struct kernfs_node *parent_kn, - struct rdt_domain *d, + struct rdt_mon_domain *d, struct rdt_resource *r, struct rdtgroup *prgrp) { - union mon_data_bits priv; - struct kernfs_node *kn; - struct mon_evt *mevt; - struct rmid_read rr; + struct kernfs_node *kn, *ckn; char name[32]; - int ret; + bool snc_mode; + int ret = 0; - sprintf(name, "mon_%s_%02d", r->name, d->id); - /* create the directory */ - kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp); - if (IS_ERR(kn)) - return PTR_ERR(kn); + lockdep_assert_held(&rdtgroup_mutex); - ret = rdtgroup_kn_set_ugid(kn); - if (ret) - goto out_destroy; + snc_mode = r->mon_scope == RESCTRL_L3_NODE; + sprintf(name, "mon_%s_%02d", r->name, snc_mode ? d->ci_id : d->hdr.id); + kn = kernfs_find_and_get(parent_kn, name); + if (kn) { + /* + * rdtgroup_mutex will prevent this directory from being + * removed. No need to keep this hold. + */ + kernfs_put(kn); + } else { + kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp); + if (IS_ERR(kn)) + return PTR_ERR(kn); - if (WARN_ON(list_empty(&r->mon.evt_list))) { - ret = -EPERM; - goto out_destroy; + ret = rdtgroup_kn_set_ugid(kn); + if (ret) + goto out_destroy; + ret = mon_add_all_files(kn, d, r, prgrp, snc_mode); + if (ret) + goto out_destroy; } - priv.u.rid = r->rid; - priv.u.domid = d->id; - list_for_each_entry(mevt, &r->mon.evt_list, list) { - priv.u.evtid = mevt->evtid; - ret = mon_addfile(kn, mevt->name, priv.priv); + if (snc_mode) { + sprintf(name, "mon_sub_%s_%02d", r->name, d->hdr.id); + ckn = kernfs_create_dir(kn, name, parent_kn->mode, prgrp); + if (IS_ERR(ckn)) { + ret = -EINVAL; + goto out_destroy; + } + + ret = rdtgroup_kn_set_ugid(ckn); if (ret) goto out_destroy; - if (resctrl_is_mbm_event(mevt->evtid)) - mon_event_read(&rr, r, d, prgrp, mevt->evtid, true); + ret = mon_add_all_files(ckn, d, r, prgrp, false); + if (ret) + goto out_destroy; } + kernfs_activate(kn); return 0; @@ -3652,7 +3137,7 @@ static int mkdir_mondata_subdir(struct kernfs_node *parent_kn, * and "monitor" groups with given domain id. */ static void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, - struct rdt_domain *d) + struct rdt_mon_domain *d) { struct kernfs_node *parent_kn; struct rdtgroup *prgrp, *crgrp; @@ -3674,13 +3159,13 @@ static int mkdir_mondata_subdir_alldom(struct kernfs_node *parent_kn, struct rdt_resource *r, struct rdtgroup *prgrp) { - struct rdt_domain *dom; + struct rdt_mon_domain *dom; int ret; /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - list_for_each_entry(dom, &r->domains, list) { + list_for_each_entry(dom, &r->mon_domains, hdr.list) { ret = mkdir_mondata_subdir(parent_kn, dom, r, prgrp); if (ret) return ret; @@ -3710,7 +3195,6 @@ static int mkdir_mondata_all(struct kernfs_node *parent_kn, struct rdtgroup *prgrp, struct kernfs_node **dest_kn) { - enum resctrl_res_level i; struct rdt_resource *r; struct kernfs_node *kn; int ret; @@ -3729,11 +3213,7 @@ static int mkdir_mondata_all(struct kernfs_node *parent_kn, * Create the subdirectories for each domain. Note that all events * in a domain like L3 are grouped into a resource whose domain is L3 */ - for (i = 0; i < RDT_NUM_RESOURCES; i++) { - r = resctrl_arch_get_resource(i); - if (!r->mon_capable) - continue; - + for_each_mon_capable_rdt_resource(r) { ret = mkdir_mondata_subdir_alldom(kn, r, prgrp); if (ret) goto out_destroy; @@ -3765,11 +3245,12 @@ static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r) { unsigned int cbm_len = r->cache.cbm_len; unsigned long first_bit, zero_bit; - unsigned long val = _val; + unsigned long val; - if (!val) - return 0; + if (!_val || r->cache.arch_has_sparse_bitmasks) + return _val; + val = _val; first_bit = find_first_bit(&val, cbm_len); zero_bit = find_next_zero_bit(&val, cbm_len, first_bit); @@ -3784,7 +3265,7 @@ static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r) * Set the RDT domain up to start off with all usable allocations. That is, * all shareable and unused bits. All-zero CBM is invalid. */ -static int __init_one_rdt_domain(struct rdt_domain *d, struct resctrl_schema *s, +static int __init_one_rdt_domain(struct rdt_ctrl_domain *d, struct resctrl_schema *s, u32 closid) { enum resctrl_conf_type peer_type = resctrl_peer_type(s->conf_type); @@ -3844,7 +3325,7 @@ static int __init_one_rdt_domain(struct rdt_domain *d, struct resctrl_schema *s, */ tmp_cbm = cfg->new_ctrl; if (bitmap_weight(&tmp_cbm, r->cache.cbm_len) < r->cache.min_cbm_bits) { - rdt_last_cmd_printf("No space on %s:%d\n", s->name, d->id); + rdt_last_cmd_printf("No space on %s:%d\n", s->name, d->hdr.id); return -ENOSPC; } cfg->have_new_ctrl = true; @@ -3862,12 +3343,12 @@ static int __init_one_rdt_domain(struct rdt_domain *d, struct resctrl_schema *s, * If there are no more shareable bits available on any domain then * the entire allocation will fail. */ -int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid) +static int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid) { - struct rdt_domain *d; + struct rdt_ctrl_domain *d; int ret; - list_for_each_entry(d, &s->res->domains, list) { + list_for_each_entry(d, &s->res->ctrl_domains, hdr.list) { ret = __init_one_rdt_domain(d, s, closid); if (ret < 0) return ret; @@ -3880,16 +3361,16 @@ int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid) static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid) { struct resctrl_staged_config *cfg; - struct rdt_domain *d; + struct rdt_ctrl_domain *d; - list_for_each_entry(d, &r->domains, list) { + list_for_each_entry(d, &r->ctrl_domains, hdr.list) { if (is_mba_sc(r)) { d->mbps_val[closid] = MBA_MAX_MBPS; continue; } cfg = &d->staged_config[CDP_NONE]; - cfg->new_ctrl = r->default_ctrl; + cfg->new_ctrl = resctrl_get_default_ctrl(r); cfg->have_new_ctrl = true; } } @@ -3921,7 +3402,6 @@ static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp) rdt_last_cmd_puts("Failed to initialize allocations\n"); goto out; } - } rdtgrp->mode = RDT_MODE_SHAREABLE; @@ -3945,9 +3425,6 @@ static int mkdir_rdt_prepare_rmid_alloc(struct rdtgroup *rdtgrp) } rdtgrp->mon.rmid = ret; - rdtgrp->mon.cntr_id[0] = MON_CNTR_UNSET; - rdtgrp->mon.cntr_id[1] = MON_CNTR_UNSET; - ret = mkdir_mondata_all(rdtgrp->kn, rdtgrp, &rdtgrp->mon.mon_data_kn); if (ret) { rdt_last_cmd_puts("kernfs subdir error\n"); @@ -3964,6 +3441,22 @@ static void mkdir_rdt_prepare_rmid_free(struct rdtgroup *rgrp) free_rmid(rgrp->closid, rgrp->mon.rmid); } +/* + * We allow creating mon groups only with in a directory called "mon_groups" + * which is present in every ctrl_mon group. Check if this is a valid + * "mon_groups" directory. + * + * 1. The directory should be named "mon_groups". + * 2. The mon group itself should "not" be named "mon_groups". + * This makes sure "mon_groups" directory always has a ctrl_mon group + * as parent. + */ +static bool is_mon_groups(struct kernfs_node *kn, const char *name) +{ + return (!strcmp(rdt_kn_name(kn), "mon_groups") && + strcmp(name, "mon_groups")); +} + static int mkdir_rdt_prepare(struct kernfs_node *parent_kn, const char *name, umode_t mode, enum rdt_group_type rtype, struct rdtgroup **r) @@ -3979,6 +3472,17 @@ static int mkdir_rdt_prepare(struct kernfs_node *parent_kn, goto out_unlock; } + rdt_last_cmd_clear(); + + /* + * Check that the parent directory for a monitor group is a "mon_groups" + * directory. + */ + if (rtype == RDTMON_GROUP && !is_mon_groups(parent_kn, name)) { + ret = -EPERM; + goto out_unlock; + } + if (rtype == RDTMON_GROUP && (prdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || prdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)) { @@ -4081,8 +3585,6 @@ static int rdtgroup_mkdir_mon(struct kernfs_node *parent_kn, goto out_unlock; } - rdtgroup_assign_cntrs(rdtgrp); - kernfs_activate(rdtgrp->kn); /* @@ -4127,8 +3629,6 @@ static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn, if (ret) goto out_closid_free; - rdtgroup_assign_cntrs(rdtgrp); - kernfs_activate(rdtgrp->kn); ret = rdtgroup_init_alloc(rdtgrp); @@ -4147,6 +3647,8 @@ static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn, rdt_last_cmd_puts("kernfs subdir error\n"); goto out_del_list; } + if (is_mba_sc(NULL)) + rdtgrp->mba_mbps_event = mba_mbps_default_event; } goto out_unlock; @@ -4154,7 +3656,6 @@ static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn, out_del_list: list_del(&rdtgrp->rdtgroup_list); out_rmid_free: - rdtgroup_unassign_cntrs(rdtgrp); mkdir_rdt_prepare_rmid_free(rdtgrp); out_closid_free: closid_free(closid); @@ -4165,22 +3666,6 @@ static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn, return ret; } -/* - * We allow creating mon groups only with in a directory called "mon_groups" - * which is present in every ctrl_mon group. Check if this is a valid - * "mon_groups" directory. - * - * 1. The directory should be named "mon_groups". - * 2. The mon group itself should "not" be named "mon_groups". - * This makes sure "mon_groups" directory always has a ctrl_mon group - * as parent. - */ -static bool is_mon_groups(struct kernfs_node *kn, const char *name) -{ - return (!strcmp(kn->name, "mon_groups") && - strcmp(name, "mon_groups")); -} - static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) { @@ -4196,11 +3681,8 @@ static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name, if (resctrl_arch_alloc_capable() && parent_kn == rdtgroup_default.kn) return rdtgroup_mkdir_ctrl_mon(parent_kn, name, mode); - /* - * If RDT monitoring is supported and the parent directory is a valid - * "mon_groups" directory, add a monitoring subdirectory. - */ - if (resctrl_arch_mon_capable() && is_mon_groups(parent_kn, name)) + /* Else, attempt to add a monitoring subdirectory. */ + if (resctrl_arch_mon_capable()) return rdtgroup_mkdir_mon(parent_kn, name, mode); return -EPERM; @@ -4215,8 +3697,11 @@ static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) /* Give any tasks back to the parent group */ rdt_move_group_tasks(rdtgrp, prdtgrp, tmpmask); - /* Update per cpu rmid of the moved CPUs first */ - closid = rdtgrp->closid; + /* + * Update per cpu closid/rmid of the moved CPUs first. + * Note: the closid will not change, but the arch code still needs it. + */ + closid = prdtgrp->closid; rmid = prdtgrp->mon.rmid; for_each_cpu(cpu, &rdtgrp->cpu_mask) resctrl_arch_set_cpu_default_closid_rmid(cpu, closid, rmid); @@ -4229,9 +3714,6 @@ static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) update_closid_rmid(tmpmask, NULL); rdtgrp->flags = RDT_DELETED; - - rdtgroup_unassign_cntrs(rdtgrp); - free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); /* @@ -4279,8 +3761,6 @@ static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask); update_closid_rmid(tmpmask, NULL); - rdtgroup_unassign_cntrs(rdtgrp); - free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); closid_free(rdtgrp->closid); @@ -4294,9 +3774,18 @@ static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) return 0; } +static struct kernfs_node *rdt_kn_parent(struct kernfs_node *kn) +{ + /* + * Valid within the RCU section it was obtained or while rdtgroup_mutex + * is held. + */ + return rcu_dereference_check(kn->__parent, lockdep_is_held(&rdtgroup_mutex)); +} + static int rdtgroup_rmdir(struct kernfs_node *kn) { - struct kernfs_node *parent_kn = kn->parent; + struct kernfs_node *parent_kn; struct rdtgroup *rdtgrp; cpumask_var_t tmpmask; int ret = 0; @@ -4309,6 +3798,7 @@ static int rdtgroup_rmdir(struct kernfs_node *kn) ret = -EPERM; goto out; } + parent_kn = rdt_kn_parent(kn); /* * If the rdtgroup is a ctrl_mon group and parent directory @@ -4326,7 +3816,7 @@ static int rdtgroup_rmdir(struct kernfs_node *kn) ret = rdtgroup_rmdir_ctrl(rdtgrp, tmpmask); } } else if (rdtgrp->type == RDTMON_GROUP && - is_mon_groups(parent_kn, kn->name)) { + is_mon_groups(parent_kn, rdt_kn_name(kn))) { ret = rdtgroup_rmdir_mon(rdtgrp, tmpmask); } else { ret = -EPERM; @@ -4377,6 +3867,7 @@ static void mongrp_reparent(struct rdtgroup *rdtgrp, static int rdtgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent, const char *new_name) { + struct kernfs_node *kn_parent; struct rdtgroup *new_prdtgrp; struct rdtgroup *rdtgrp; cpumask_var_t tmpmask; @@ -4411,8 +3902,9 @@ static int rdtgroup_rename(struct kernfs_node *kn, goto out; } - if (rdtgrp->type != RDTMON_GROUP || !kn->parent || - !is_mon_groups(kn->parent, kn->name)) { + kn_parent = rdt_kn_parent(kn); + if (rdtgrp->type != RDTMON_GROUP || !kn_parent || + !is_mon_groups(kn_parent, rdt_kn_name(kn))) { rdt_last_cmd_puts("Source must be a MON group\n"); ret = -EPERM; goto out; @@ -4476,9 +3968,6 @@ static int rdtgroup_show_options(struct seq_file *seq, struct kernfs_root *kf) if (is_mba_sc(resctrl_arch_get_resource(RDT_RESOURCE_MBA))) seq_puts(seq, ",mba_MBps"); - if (is_hwdrc_enabled(resctrl_arch_get_resource(RDT_RESOURCE_MBA))) - seq_puts(seq, ",hwdrc_mb"); - if (resctrl_debug) seq_puts(seq, ",debug"); @@ -4509,6 +3998,8 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx) static void rdtgroup_destroy_root(void) { + lockdep_assert_held(&rdtgroup_mutex); + kernfs_destroy_root(rdt_root); rdtgroup_default.kn = NULL; } @@ -4520,9 +4011,6 @@ static void rdtgroup_setup_default(void) rdtgroup_default.closid = RESCTRL_RESERVED_CLOSID; rdtgroup_default.mon.rmid = RESCTRL_RESERVED_RMID; rdtgroup_default.type = RDTCTRL_GROUP; - rdtgroup_default.mon.cntr_id[0] = MON_CNTR_UNSET; - rdtgroup_default.mon.cntr_id[1] = MON_CNTR_UNSET; - INIT_LIST_HEAD(&rdtgroup_default.mon.crdtgrp_list); list_add(&rdtgroup_default.rdtgroup_list, &rdt_all_groups); @@ -4530,30 +4018,33 @@ static void rdtgroup_setup_default(void) mutex_unlock(&rdtgroup_mutex); } -static void domain_destroy_mon_state(struct rdt_domain *d) +static void domain_destroy_mon_state(struct rdt_mon_domain *d) { - bitmap_free(d->mbm_cntr_map); bitmap_free(d->rmid_busy_llc); kfree(d->mbm_total); kfree(d->mbm_local); } -void resctrl_offline_domain(struct rdt_resource *r, struct rdt_domain *d) +void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d) { mutex_lock(&rdtgroup_mutex); if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) mba_sc_domain_destroy(r, d); - if (!r->mon_capable) - goto out_unlock; + mutex_unlock(&rdtgroup_mutex); +} + +void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d) +{ + mutex_lock(&rdtgroup_mutex); /* * If resctrl is mounted, remove all the * per domain monitor data directories. */ if (resctrl_mounted && resctrl_arch_mon_capable()) - rmdir_mondata_subdir_allrdtgrp(r, d->id); + rmdir_mondata_subdir_allrdtgrp(r, d); if (resctrl_is_mbm_enabled()) cancel_delayed_work(&d->mbm_over); @@ -4572,11 +4063,23 @@ void resctrl_offline_domain(struct rdt_resource *r, struct rdt_domain *d) domain_destroy_mon_state(d); -out_unlock: mutex_unlock(&rdtgroup_mutex); } -static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_domain *d) +/** + * domain_setup_mon_state() - Initialise domain monitoring structures. + * @r: The resource for the newly online domain. + * @d: The newly online domain. + * + * Allocate monitor resources that belong to this domain. + * Called when the first CPU of a domain comes online, regardless of whether + * the filesystem is mounted. + * During boot this may be called before global allocations have been made by + * resctrl_mon_resource_init(). + * + * Returns 0 for success, or -ENOMEM. + */ +static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_mon_domain *d) { u32 idx_limit = resctrl_arch_system_num_rmid_idx(); size_t tsize; @@ -4603,20 +4106,11 @@ static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_domain *d) return -ENOMEM; } } - if (resctrl_is_mbm_enabled()) { - d->mbm_cntr_map = bitmap_zalloc(r->mon.num_mbm_cntrs, GFP_KERNEL); - if (!d->mbm_cntr_map) { - bitmap_free(d->rmid_busy_llc); - kfree(d->mbm_total); - kfree(d->mbm_local); - return -ENOMEM; - } - } return 0; } -int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d) +int resctrl_online_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d) { int err = 0; @@ -4625,11 +4119,18 @@ int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d) if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) { /* RDT_RESOURCE_MBA is never mon_capable */ err = mba_sc_domain_allocate(r, d); - goto out_unlock; } - if (!r->mon_capable) - goto out_unlock; + mutex_unlock(&rdtgroup_mutex); + + return err; +} + +int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d) +{ + int err; + + mutex_lock(&rdtgroup_mutex); err = domain_setup_mon_state(r, d); if (err) @@ -4677,11 +4178,27 @@ static void clear_childcpus(struct rdtgroup *r, unsigned int cpu) } } +static struct rdt_mon_domain *get_mon_domain_from_cpu(int cpu, + struct rdt_resource *r) +{ + struct rdt_mon_domain *d; + + lockdep_assert_cpus_held(); + + list_for_each_entry(d, &r->mon_domains, hdr.list) { + /* Find the domain that contains this CPU */ + if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) + return d; + } + + return NULL; +} + void resctrl_offline_cpu(unsigned int cpu) { struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_mon_domain *d; struct rdtgroup *rdtgrp; - struct rdt_domain *d; mutex_lock(&rdtgroup_mutex); list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { @@ -4694,7 +4211,7 @@ void resctrl_offline_cpu(unsigned int cpu) if (!l3->mon_capable) goto out_unlock; - d = resctrl_get_domain_from_cpu(cpu, l3); + d = get_mon_domain_from_cpu(cpu, l3); if (d) { if (resctrl_is_mbm_enabled() && cpu == d->mbm_work_cpu) { cancel_delayed_work(&d->mbm_over); @@ -4728,19 +4245,17 @@ int resctrl_init(void) rdtgroup_setup_default(); - resctrl_file_fflags_init("thread_throttle_mode", - RFTYPE_CTRL_INFO | RFTYPE_RES_MB); -#ifdef CONFIG_X86 - io_alloc_init(); -#endif + thread_throttle_mode_init(); ret = resctrl_mon_resource_init(); if (ret) return ret; ret = sysfs_create_mount_point(fs_kobj, "resctrl"); - if (ret) + if (ret) { + resctrl_mon_resource_exit(); return ret; + } ret = register_filesystem(&rdt_fs_type); if (ret) @@ -4773,15 +4288,68 @@ int resctrl_init(void) cleanup_mountpoint: sysfs_remove_mount_point(fs_kobj, "resctrl"); + resctrl_mon_resource_exit(); return ret; } +static bool resctrl_online_domains_exist(void) +{ + struct rdt_resource *r; + + /* + * Only walk capable resources to allow resctrl_arch_get_resource() + * to return dummy 'not capable' resources. + */ + for_each_alloc_capable_rdt_resource(r) { + if (!list_empty(&r->ctrl_domains)) + return true; + } + + for_each_mon_capable_rdt_resource(r) { + if (!list_empty(&r->mon_domains)) + return true; + } + + return false; +} + +/** + * resctrl_exit() - Remove the resctrl filesystem and free resources. + * + * Called by the architecture code in response to a fatal error. + * Removes resctrl files and structures from kernfs to prevent further + * configuration. + * + * When called by the architecture code, all CPUs and resctrl domains must be + * offline. This ensures the limbo and overflow handlers are not scheduled to + * run, meaning the data structures they access can be freed by + * resctrl_mon_resource_exit(). + * + * After resctrl_exit() returns, the architecture code should return an + * error from all resctrl_arch_ functions that can do this. + * resctrl_arch_get_resource() must continue to return struct rdt_resources + * with the correct rid field to ensure the filesystem can be unmounted. + */ void resctrl_exit(void) { + cpus_read_lock(); + WARN_ON_ONCE(resctrl_online_domains_exist()); + + mutex_lock(&rdtgroup_mutex); + resctrl_fs_teardown(); + mutex_unlock(&rdtgroup_mutex); + + cpus_read_unlock(); + debugfs_remove_recursive(debugfs_resctrl); + debugfs_resctrl = NULL; unregister_filesystem(&rdt_fs_type); - sysfs_remove_mount_point(fs_kobj, "resctrl"); + + /* + * Do not remove the sysfs mount point added by resctrl_init() so that + * it can be used to umount resctrl. + */ resctrl_mon_resource_exit(); } diff --git a/fs/sysfs/dir.c b/fs/sysfs/dir.c index b6b6796e16160dbdc92e1f63d370acba93b82760..2e7b9cb2788e2bbafa78c6255c0e2a53fa4313cd 100644 --- a/fs/sysfs/dir.c +++ b/fs/sysfs/dir.c @@ -123,7 +123,7 @@ int sysfs_move_dir_ns(struct kobject *kobj, struct kobject *new_parent_kobj, new_parent = new_parent_kobj && new_parent_kobj->sd ? new_parent_kobj->sd : sysfs_root_kn; - return kernfs_rename_ns(kn, new_parent, kn->name, new_ns); + return kernfs_rename_ns(kn, new_parent, NULL, new_ns); } /** diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c index f536840ee5b79829d36cf1cc11c379a8f4218b07..a6390ff32210d8fc9f9e3d701d47c0434d13386d 100644 --- a/fs/sysfs/file.c +++ b/fs/sysfs/file.c @@ -19,13 +19,19 @@ #include "sysfs.h" +static struct kobject *sysfs_file_kobj(struct kernfs_node *kn) +{ + guard(rcu)(); + return rcu_dereference(kn->__parent)->priv; +} + /* * Determine ktype->sysfs_ops for the given kernfs_node. This function * must be called while holding an active reference. */ static const struct sysfs_ops *sysfs_file_ops(struct kernfs_node *kn) { - struct kobject *kobj = kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(kn); if (kn->flags & KERNFS_LOCKDEP) lockdep_assert_held(kn); @@ -40,7 +46,7 @@ static const struct sysfs_ops *sysfs_file_ops(struct kernfs_node *kn) static int sysfs_kf_seq_show(struct seq_file *sf, void *v) { struct kernfs_open_file *of = sf->private; - struct kobject *kobj = of->kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); const struct sysfs_ops *ops = sysfs_file_ops(of->kn); ssize_t count; char *buf; @@ -78,7 +84,7 @@ static ssize_t sysfs_kf_bin_read(struct kernfs_open_file *of, char *buf, size_t count, loff_t pos) { struct bin_attribute *battr = of->kn->priv; - struct kobject *kobj = of->kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); loff_t size = file_inode(of->file)->i_size; if (!count) @@ -105,7 +111,7 @@ static ssize_t sysfs_kf_read(struct kernfs_open_file *of, char *buf, size_t count, loff_t pos) { const struct sysfs_ops *ops = sysfs_file_ops(of->kn); - struct kobject *kobj = of->kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); ssize_t len; /* @@ -131,7 +137,7 @@ static ssize_t sysfs_kf_write(struct kernfs_open_file *of, char *buf, size_t count, loff_t pos) { const struct sysfs_ops *ops = sysfs_file_ops(of->kn); - struct kobject *kobj = of->kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); if (!count) return 0; @@ -144,7 +150,7 @@ static ssize_t sysfs_kf_bin_write(struct kernfs_open_file *of, char *buf, size_t count, loff_t pos) { struct bin_attribute *battr = of->kn->priv; - struct kobject *kobj = of->kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); loff_t size = file_inode(of->file)->i_size; if (size) { @@ -168,11 +174,23 @@ static int sysfs_kf_bin_mmap(struct kernfs_open_file *of, struct vm_area_struct *vma) { struct bin_attribute *battr = of->kn->priv; - struct kobject *kobj = of->kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); return battr->mmap(of->file, kobj, battr, vma); } +static loff_t sysfs_kf_bin_llseek(struct kernfs_open_file *of, loff_t offset, + int whence) +{ + struct bin_attribute *battr = of->kn->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); + + if (battr->llseek) + return battr->llseek(of->file, kobj, battr, offset, whence); + else + return generic_file_llseek(of->file, offset, whence); +} + static int sysfs_kf_bin_open(struct kernfs_open_file *of) { struct bin_attribute *battr = of->kn->priv; @@ -255,6 +273,7 @@ static const struct kernfs_ops sysfs_bin_kfops_mmap = { .write = sysfs_kf_bin_write, .mmap = sysfs_kf_bin_mmap, .open = sysfs_kf_bin_open, + .llseek = sysfs_kf_bin_llseek, }; int sysfs_add_file_mode_ns(struct kernfs_node *parent, @@ -481,7 +500,7 @@ EXPORT_SYMBOL_GPL(sysfs_break_active_protection); */ void sysfs_unbreak_active_protection(struct kernfs_node *kn) { - struct kobject *kobj = kn->parent->priv; + struct kobject *kobj = sysfs_file_kobj(kn); kernfs_unbreak_active_protection(kn); kernfs_put(kn); diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b339abbfa2a4841599976e6ab89d11e0364e5188..8ac16b6967d8d6119da7a75b1ec83534f57d06b1 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1493,11 +1493,6 @@ int find_acpi_cpu_topology(unsigned int cpu, int level); int find_acpi_cpu_topology_cluster(unsigned int cpu); int find_acpi_cpu_topology_package(unsigned int cpu); int find_acpi_cpu_topology_hetero_id(unsigned int cpu); -int find_acpi_cache_level_from_id(u32 cache_id); -int acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id, cpumask_t *cpus); -int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, cpumask_t *cpus); -int acpi_pptt_get_cpumask_from_cache_id_and_level(u32 cache_id, u32 cache_level, - cpumask_t *cpus); #else static inline int acpi_pptt_cpu_is_thread(unsigned int cpu) { @@ -1519,26 +1514,6 @@ static inline int find_acpi_cpu_topology_hetero_id(unsigned int cpu) { return -EINVAL; } -static inline int find_acpi_cache_level_from_id(u32 cache_id) -{ - return -EINVAL; -} -static inline int acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id, - cpumask_t *cpus) -{ - return -EINVAL; -} -static inline int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, - cpumask_t *cpus) -{ - return -EINVAL; -} -static inline int acpi_pptt_get_cpumask_from_cache_id_and_level(u32 cache_id, - u32 cache_level, - cpumask_t *cpus) -{ - return -EINVAL; -} #endif #ifdef CONFIG_ARM64 @@ -1577,8 +1552,4 @@ static inline void acpi_device_notify(struct device *dev) { } static inline void acpi_device_notify_remove(struct device *dev) { } #endif -struct acpi_pptt_processor * -acpi_pptt_find_cache_backwards(struct acpi_table_header *table_hdr, - struct acpi_pptt_cache *cache); - #endif /*_LINUX_ACPI_H*/ diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h deleted file mode 100644 index c7698dda09fb86d1598cf0bbb94a64d230572ab9..0000000000000000000000000000000000000000 --- a/include/linux/arm_mpam.h +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2021 Arm Ltd. - -#ifndef __LINUX_ARM_MPAM_H -#define __LINUX_ARM_MPAM_H - -#include -#include -#include - -/* - * The value of the MPAM1_EL1 sysreg when a task is in the default group. - * This is used by the context switch code to use the resctrl CPU property - * instead. The value is modified when CDP is enabled/disabled by mounting - * the resctrl filesystem. - */ -extern u64 mpam_resctrl_default_group; - -#include - -struct mpam_msc; - -enum mpam_msc_iface { - MPAM_IFACE_MMIO, /* a real MPAM MSC */ - MPAM_IFACE_PCC, /* a fake MPAM MSC */ -}; - -enum mpam_class_types { - MPAM_CLASS_CACHE, /* Well known caches, e.g. L2 */ - MPAM_CLASS_MEMORY, /* Main memory */ - MPAM_CLASS_UNKNOWN, /* Everything else, e.g. SMMU */ -}; - -enum mpam_machine_type { - MPAM_DEFAULT_MACHINE, - MPAM_YITIAN710, - - MPAM_NUM_MACHINE_TYPES, -}; - -/* Machine identifier which can be used for vendor-specific MPAM features */ -extern enum mpam_machine_type mpam_current_machine; - -#ifdef CONFIG_ACPI_MPAM -/* Parse the ACPI description of resources entries for this MSC. */ -int acpi_mpam_parse_resources(struct mpam_msc *msc, - struct acpi_mpam_msc_node *tbl_msc); -int acpi_mpam_count_msc(void); -enum mpam_machine_type acpi_mpam_get_machine_type(void); -#else -static inline int acpi_mpam_parse_resources(struct mpam_msc *msc, - struct acpi_mpam_msc_node *tbl_msc) -{ - return -EINVAL; -} -static inline int acpi_mpam_count_msc(void) { return -EINVAL; } -static inline enum mpam_machine_type acpi_mpam_get_machine_type(void) -{ - return MPAM_DEFAULT_MACHINE; -} -#endif - -int mpam_register_requestor(u16 partid_max, u8 pmg_max); - -int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, - enum mpam_class_types type, u8 class_id, int component_id); - -static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) -{ - return val; -} - -/* MPAM counters requires a monitor to be allocated */ -static inline bool resctrl_arch_event_is_free_running(enum resctrl_event_id evt) -{ - return false; -} - -bool resctrl_arch_alloc_capable(void); -bool resctrl_arch_mon_capable(void); -bool resctrl_arch_is_llc_occupancy_enabled(void); -bool resctrl_arch_is_mbm_local_enabled(void); -bool resctrl_arch_is_mbm_total_enabled(void); -bool resctrl_arch_is_mbm_bps_enabled(void); - -static inline bool resctrl_arch_is_hwdrc_mb_capable(void) -{ - return false; -} - -static inline int resctrl_arch_set_hwdrc_enabled(enum resctrl_res_level ignored, bool hwdrc_mb) -{ - return hwdrc_mb ? -EINVAL : 0; -} - -/* reset cached configurations, then all devices */ -void resctrl_arch_reset_resources(void); - -bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level ignored); -int resctrl_arch_set_cdp_enabled(enum resctrl_res_level ignored, bool enable); -bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid); -bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid); -void resctrl_arch_set_cpu_default_closid(int cpu, u32 closid); -void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid); -void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 pmg); -void resctrl_arch_sched_in(struct task_struct *tsk); -u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid); -void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid); -u32 resctrl_arch_system_num_rmid_idx(void); - -struct rdt_resource; -void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, int evtid); -void resctrl_arch_mon_ctx_free(struct rdt_resource *r, int evtid, void *ctx); - -/* Pseudo lock is not supported by MPAM */ -static inline int resctrl_arch_pseudo_lock_fn(void *_plr) { return 0; } -static inline int resctrl_arch_measure_l2_residency(void *_plr) { return 0; } -static inline int resctrl_arch_measure_l3_residency(void *_plr) { return 0; } -static inline int resctrl_arch_measure_cycles_lat_fn(void *_plr) { return 0; } -static inline u64 resctrl_arch_get_prefetch_disable_bits(void) { return 0; } - -/* - * The CPU configuration for MPAM is cheap to write, and is only written if it - * has changed. No need for fine grained enables. - */ -static inline void resctrl_arch_enable_mon(void) { } -static inline void resctrl_arch_disable_mon(void) { } -static inline void resctrl_arch_enable_alloc(void) { } -static inline void resctrl_arch_disable_alloc(void) { } - -static inline void resctrl_arch_mbm_cntr_assign_configure(void) { } - -static inline bool resctrl_arch_get_abmc_enabled(void) -{ - return false; -} - -static inline int resctrl_arch_mbm_cntr_assign_enable(void) -{ - return -EINVAL; -} - -static inline void resctrl_arch_mbm_cntr_assign_disable(void) { } - -void resctrl_arch_event_config_set(void *info); -u32 resctrl_arch_event_config_get(void *dom, - enum resctrl_event_id eventid); - -static inline bool resctrl_arch_get_mbm_cntr_assign_enable(void) -{ - return false; -} - -static inline int resctrl_arch_assign_cntr(void *dom, enum resctrl_event_id evtid, - u32 rmid, u32 cntr_id, u32 closid, bool assign) -{ - return -EINVAL; -} - -#endif /* __LINUX_ARM_MPAM_H */ diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 850ecce05de9d35824bcbe46c12dae85675f9e7e..6100b05c943f747fc66dc076bbf087f0c01a7e35 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -53,6 +53,7 @@ struct device; * bitmap_full(src, nbits) Are all bits set in *src? * bitmap_weight(src, nbits) Hamming Weight: number set bits * bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap + * bitmap_weight_andnot(src1, src2, nbits) Hamming Weight of andnot'ed bitmap * bitmap_set(dst, pos, nbits) Set specified bit area * bitmap_clear(dst, pos, nbits) Clear specified bit area * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area @@ -174,6 +175,8 @@ bool __bitmap_subset(const unsigned long *bitmap1, unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); unsigned int __bitmap_weight_and(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); +unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int nbits); void __bitmap_set(unsigned long *map, unsigned int start, int len); void __bitmap_clear(unsigned long *map, unsigned int start, int len); @@ -469,6 +472,15 @@ unsigned long bitmap_weight_and(const unsigned long *src1, return __bitmap_weight_and(src1, src2, nbits); } +static __always_inline +unsigned long bitmap_weight_andnot(const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) +{ + if (small_const_nbits(nbits)) + return hweight_long(*src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)); + return __bitmap_weight_andnot(src1, src2, nbits); +} + static __always_inline void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits) { diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index 251e8b393ec7282caf889438b6999fc74a297735..ef701ba043e9407b82d54152e49bf36b9f516ec9 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -3,6 +3,7 @@ #define _LINUX_CACHEINFO_H #include +#include #include #include @@ -47,7 +48,7 @@ extern unsigned int coherency_max_size; * keeping, the remaining members form the core properties of the cache */ struct cacheinfo { - unsigned long id; + unsigned int id; enum cache_type type; unsigned int level; unsigned int coherency_line_size; @@ -111,47 +112,39 @@ int acpi_get_cache_info(unsigned int cpu, #endif const struct attribute_group *cache_get_priv_group(struct cacheinfo *this_leaf); -unsigned long cache_of_get_id(struct device_node *np); /* - * Get the id of the cache associated with @cpu at level @level. + * Get the cacheinfo structure for the cache associated with @cpu at + * level @level. * cpuhp lock must be held. */ -static inline unsigned long get_cpu_cacheinfo_id(int cpu, int level) +static inline struct cacheinfo *get_cpu_cacheinfo_level(int cpu, int level) { struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); int i; + lockdep_assert_cpus_held(); + for (i = 0; i < ci->num_leaves; i++) { if (ci->info_list[i].level == level) { if (ci->info_list[i].attributes & CACHE_ID) - return ci->info_list[i].id; - return ~0UL; + return &ci->info_list[i]; + return NULL; } } - return ~0UL; + return NULL; } /* - * Get the size of the cache associated with @cpu at level @level. + * Get the id of the cache associated with @cpu at level @level. * cpuhp lock must be held. */ -static inline unsigned int get_cpu_cacheinfo_size(int cpu, int level) +static inline int get_cpu_cacheinfo_id(int cpu, int level) { - struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); - int i; - - if (!ci->info_list) - return 0; - - for (i = 0; i < ci->num_leaves; i++) { - if (ci->info_list[i].level == level) { - return ci->info_list[i].size; - } - } + struct cacheinfo *ci = get_cpu_cacheinfo_level(cpu, level); - return 0; + return ci ? ci->id : -1; } #ifdef CONFIG_ARM64 diff --git a/include/linux/cpu.h b/include/linux/cpu.h index b9900a4d488472715671a60da093777dc213eeba..e397c102cebe4df38117aefaf13191e7a8972b53 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -18,6 +18,7 @@ #include #include #include +#include #include struct device; @@ -133,38 +134,6 @@ static inline int arch_cpu_rescan_dead_smt_siblings(void) { return 0; } #endif /* CONFIG_SMP */ extern struct bus_type cpu_subsys; -extern int lockdep_is_cpus_held(void); - -#ifdef CONFIG_HOTPLUG_CPU -extern void cpus_write_lock(void); -extern void cpus_write_unlock(void); -extern void cpus_read_lock(void); -extern void cpus_read_unlock(void); -extern int cpus_read_trylock(void); -extern void lockdep_assert_cpus_held(void); -extern void cpu_hotplug_disable(void); -extern void cpu_hotplug_enable(void); -void clear_tasks_mm_cpumask(int cpu); -int remove_cpu(unsigned int cpu); -int cpu_device_down(struct device *dev); -extern void smp_shutdown_nonboot_cpus(unsigned int primary_cpu); - -#else /* CONFIG_HOTPLUG_CPU */ - -static inline void cpus_write_lock(void) { } -static inline void cpus_write_unlock(void) { } -static inline void cpus_read_lock(void) { } -static inline void cpus_read_unlock(void) { } -static inline int cpus_read_trylock(void) { return true; } -static inline void lockdep_assert_cpus_held(void) { } -static inline void cpu_hotplug_disable(void) { } -static inline void cpu_hotplug_enable(void) { } -static inline int remove_cpu(unsigned int cpu) { return -EPERM; } -static inline void smp_shutdown_nonboot_cpus(unsigned int primary_cpu) { } -#endif /* !CONFIG_HOTPLUG_CPU */ - -DEFINE_LOCK_GUARD_0(cpus_read_lock, cpus_read_lock(), cpus_read_unlock()) - #ifdef CONFIG_PM_SLEEP_SMP extern int freeze_secondary_cpus(int primary); extern void thaw_secondary_cpus(void); diff --git a/include/linux/cpuhplock.h b/include/linux/cpuhplock.h new file mode 100644 index 0000000000000000000000000000000000000000..431560bbd0453d7b3fce9726e70dcb6057ddcf6b --- /dev/null +++ b/include/linux/cpuhplock.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/linux/cpuhplock.h - CPU hotplug locking + * + * Locking functions for CPU hotplug. + */ +#ifndef _LINUX_CPUHPLOCK_H_ +#define _LINUX_CPUHPLOCK_H_ + +#include +#include + +struct device; + +extern int lockdep_is_cpus_held(void); + +#ifdef CONFIG_HOTPLUG_CPU +void cpus_write_lock(void); +void cpus_write_unlock(void); +void cpus_read_lock(void); +void cpus_read_unlock(void); +int cpus_read_trylock(void); +void lockdep_assert_cpus_held(void); +void cpu_hotplug_disable(void); +void cpu_hotplug_enable(void); +void clear_tasks_mm_cpumask(int cpu); +int remove_cpu(unsigned int cpu); +int cpu_device_down(struct device *dev); +void smp_shutdown_nonboot_cpus(unsigned int primary_cpu); + +#else /* CONFIG_HOTPLUG_CPU */ + +static inline void cpus_write_lock(void) { } +static inline void cpus_write_unlock(void) { } +static inline void cpus_read_lock(void) { } +static inline void cpus_read_unlock(void) { } +static inline int cpus_read_trylock(void) { return true; } +static inline void lockdep_assert_cpus_held(void) { } +static inline void cpu_hotplug_disable(void) { } +static inline void cpu_hotplug_enable(void) { } +static inline int remove_cpu(unsigned int cpu) { return -EPERM; } +static inline void smp_shutdown_nonboot_cpus(unsigned int primary_cpu) { } +#endif /* !CONFIG_HOTPLUG_CPU */ + +DEFINE_LOCK_GUARD_0(cpus_read_lock, cpus_read_lock(), cpus_read_unlock()) + +#endif /* _LINUX_CPUHPLOCK_H_ */ diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index cb4850e09e43ce922ddb9b52663b0395a9973b90..afd540f321012fff8e9c3db00ebe5c7d77cc440b 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -4,7 +4,7 @@ /* * Cpumasks provide a bitmap suitable for representing the - * set of CPU's in a system, one bit position per CPU number. In general, + * set of CPUs in a system, one bit position per CPU number. In general, * only nr_cpu_ids (<= NR_CPUS) bits are valid. */ #include @@ -41,7 +41,7 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; extern unsigned int nr_cpu_ids; #endif -static inline void set_nr_cpu_ids(unsigned int nr) +static __always_inline void set_nr_cpu_ids(unsigned int nr) { #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) WARN_ON(nr != nr_cpu_ids); @@ -97,7 +97,7 @@ static inline void set_nr_cpu_ids(unsigned int nr) * * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. * - * The cpu_possible_mask is fixed at boot time, as the set of CPU id's + * The cpu_possible_mask is fixed at boot time, as the set of CPU IDs * that it is possible might ever be plugged in at anytime during the * life of that system boot. The cpu_present_mask is dynamic(*), * representing which CPUs are currently plugged in. And @@ -112,7 +112,7 @@ static inline void set_nr_cpu_ids(unsigned int nr) * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot. * * Subtleties: - * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode + * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode * assumption that their single CPU is online. The UP * cpu_{online,possible,present}_masks are placebos. Changing them * will have no useful affect on the following num_*_cpus() @@ -155,9 +155,9 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu) * cpumask_first - get the first cpu in a cpumask * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if no cpus set. + * Return: >= nr_cpu_ids if no cpus set. */ -static inline unsigned int cpumask_first(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_first(const struct cpumask *srcp) { return find_first_bit(cpumask_bits(srcp), small_cpumask_bits); } @@ -166,9 +166,9 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) * cpumask_first_zero - get the first unset cpu in a cpumask * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if all cpus are set. + * Return: >= nr_cpu_ids if all cpus are set. */ -static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp) { return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits); } @@ -178,14 +178,27 @@ static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) * @srcp1: the first input * @srcp2: the second input * - * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). + * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). */ -static inline +static __always_inline unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) { return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); } +/** + * cpumask_first_andnot - return the first cpu from *srcp1 & ~*srcp2 + * @srcp1: the first input + * @srcp2: the second input + * + * Return: >= nr_cpu_ids if no such cpu found. + */ +static __always_inline +unsigned int cpumask_first_andnot(const struct cpumask *srcp1, const struct cpumask *srcp2) +{ + return find_first_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); +} + /** * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3 * @srcp1: the first input @@ -194,7 +207,7 @@ unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask * * Return: >= nr_cpu_ids if no cpus set in all. */ -static inline +static __always_inline unsigned int cpumask_first_and_and(const struct cpumask *srcp1, const struct cpumask *srcp2, const struct cpumask *srcp3) @@ -207,21 +220,21 @@ unsigned int cpumask_first_and_and(const struct cpumask *srcp1, * cpumask_last - get the last CPU in a cpumask * @srcp: - the cpumask pointer * - * Returns >= nr_cpumask_bits if no CPUs set. + * Return: >= nr_cpumask_bits if no CPUs set. */ -static inline unsigned int cpumask_last(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_last(const struct cpumask *srcp) { return find_last_bit(cpumask_bits(srcp), small_cpumask_bits); } /** * cpumask_next - get the next cpu in a cpumask - * @n: the cpu prior to the place to search (ie. return will be > @n) + * @n: the cpu prior to the place to search (i.e. return will be > @n) * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if no further cpus set. + * Return: >= nr_cpu_ids if no further cpus set. */ -static inline +static __always_inline unsigned int cpumask_next(int n, const struct cpumask *srcp) { /* -1 is a legal arg here. */ @@ -232,12 +245,13 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp) /** * cpumask_next_zero - get the next unset cpu in a cpumask - * @n: the cpu prior to the place to search (ie. return will be > @n) + * @n: the cpu prior to the place to search (i.e. return will be > @n) * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if no further cpus unset. + * Return: >= nr_cpu_ids if no further cpus unset. */ -static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) +static __always_inline +unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) { /* -1 is a legal arg here. */ if (n != -1) @@ -247,18 +261,21 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) #if NR_CPUS == 1 /* Uniprocessor: there is only one valid CPU */ -static inline unsigned int cpumask_local_spread(unsigned int i, int node) +static __always_inline +unsigned int cpumask_local_spread(unsigned int i, int node) { return 0; } -static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, + const struct cpumask *src2p) { return cpumask_first_and(src1p, src2p); } -static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp) +static __always_inline +unsigned int cpumask_any_distribute(const struct cpumask *srcp) { return cpumask_first(srcp); } @@ -271,15 +288,15 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp); /** * cpumask_next_and - get the next cpu in *src1p & *src2p - * @n: the cpu prior to the place to search (ie. return will be > @n) + * @n: the cpu prior to the place to search (i.e. return will be > @n) * @src1p: the first cpumask pointer * @src2p: the second cpumask pointer * - * Returns >= nr_cpu_ids if no further cpus set in both. + * Return: >= nr_cpu_ids if no further cpus set in both. */ -static inline +static __always_inline unsigned int cpumask_next_and(int n, const struct cpumask *src1p, - const struct cpumask *src2p) + const struct cpumask *src2p) { /* -1 is a legal arg here. */ if (n != -1) @@ -288,6 +305,25 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, small_cpumask_bits, n + 1); } +/** + * cpumask_next_andnot - get the next cpu in *src1p & ~*src2p + * @n: the cpu prior to the place to search (i.e. return will be > @n) + * @src1p: the first cpumask pointer + * @src2p: the second cpumask pointer + * + * Return: >= nr_cpu_ids if no further cpus set in both. + */ +static __always_inline +unsigned int cpumask_next_andnot(int n, const struct cpumask *src1p, + const struct cpumask *src2p) +{ + /* -1 is a legal arg here. */ + if (n != -1) + cpumask_check(n); + return find_next_andnot_bit(cpumask_bits(src1p), cpumask_bits(src2p), + small_cpumask_bits, n + 1); +} + /** * for_each_cpu - iterate over every cpu in a mask * @cpu: the (optionally unsigned) integer iterator @@ -299,7 +335,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits) #if NR_CPUS == 1 -static inline +static __always_inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) { cpumask_check(start); @@ -390,14 +426,18 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta * @cpu: the cpu to ignore. * * Often used to find any cpu but smp_processor_id() in a mask. - * Returns >= nr_cpu_ids if no cpus set. + * If @cpu == -1, the function is equivalent to cpumask_any(). + * Return: >= nr_cpu_ids if no cpus set. */ -static inline -unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) +static __always_inline +unsigned int cpumask_any_but(const struct cpumask *mask, int cpu) { unsigned int i; - cpumask_check(cpu); + /* -1 is a legal arg here. */ + if (cpu != -1) + cpumask_check(cpu); + for_each_cpu(i, mask) if (i != cpu) break; @@ -410,16 +450,20 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) * @mask2: the second input cpumask * @cpu: the cpu to ignore * + * If @cpu == -1, the function is equivalent to cpumask_any_and(). * Returns >= nr_cpu_ids if no cpus set. */ -static inline +static __always_inline unsigned int cpumask_any_and_but(const struct cpumask *mask1, const struct cpumask *mask2, - unsigned int cpu) + int cpu) { unsigned int i; - cpumask_check(cpu); + /* -1 is a legal arg here. */ + if (cpu != -1) + cpumask_check(cpu); + i = cpumask_first_and(mask1, mask2); if (i != cpu) return i; @@ -427,27 +471,55 @@ unsigned int cpumask_any_and_but(const struct cpumask *mask1, return cpumask_next_and(cpu, mask1, mask2); } +/** + * cpumask_any_andnot_but - pick an arbitrary cpu from *mask1 & ~*mask2, but not this one. + * @mask1: the first input cpumask + * @mask2: the second input cpumask + * @cpu: the cpu to ignore + * + * If @cpu == -1, the function returns the first matching cpu. + * Returns >= nr_cpu_ids if no cpus set. + */ +static __always_inline +unsigned int cpumask_any_andnot_but(const struct cpumask *mask1, + const struct cpumask *mask2, + int cpu) +{ + unsigned int i; + + /* -1 is a legal arg here. */ + if (cpu != -1) + cpumask_check(cpu); + + i = cpumask_first_andnot(mask1, mask2); + if (i != cpu) + return i; + + return cpumask_next_andnot(cpu, mask1, mask2); +} + /** * cpumask_nth - get the Nth cpu in a cpumask * @srcp: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ -static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) +static __always_inline +unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) { return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu)); } /** - * cpumask_nth_and - get the first cpu in 2 cpumasks + * cpumask_nth_and - get the Nth cpu in 2 cpumasks * @srcp1: the cpumask pointer * @srcp2: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ -static inline +static __always_inline unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, const struct cpumask *srcp2) { @@ -456,14 +528,14 @@ unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, } /** - * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd. + * cpumask_nth_andnot - get the Nth cpu set in 1st cpumask, and clear in 2nd. * @srcp1: the cpumask pointer * @srcp2: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ -static inline +static __always_inline unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, const struct cpumask *srcp2) { @@ -476,9 +548,9 @@ unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, * @srcp1: the cpumask pointer * @srcp2: the cpumask pointer * @srcp3: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ static __always_inline unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1, @@ -506,12 +578,14 @@ unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp * @cpu: cpu number (< nr_cpu_ids) * @dstp: the cpumask pointer */ -static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +static __always_inline +void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) { set_bit(cpumask_check(cpu), cpumask_bits(dstp)); } -static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +static __always_inline +void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) { __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); } @@ -537,9 +611,10 @@ static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns true if @cpu is set in @cpumask, else returns false + * Return: true if @cpu is set in @cpumask, else returns false */ -static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) +static __always_inline +bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) { return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); } @@ -549,11 +624,12 @@ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpum * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns true if @cpu is set in old bitmap of @cpumask, else returns false - * * test_and_set_bit wrapper for cpumasks. + * + * Return: true if @cpu is set in old bitmap of @cpumask, else returns false */ -static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) +static __always_inline +bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) { return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -563,11 +639,12 @@ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cp * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns true if @cpu is set in old bitmap of @cpumask, else returns false - * * test_and_clear_bit wrapper for cpumasks. + * + * Return: true if @cpu is set in old bitmap of @cpumask, else returns false */ -static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) +static __always_inline +bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) { return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -576,7 +653,7 @@ static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask * * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask * @dstp: the cpumask pointer */ -static inline void cpumask_setall(struct cpumask *dstp) +static __always_inline void cpumask_setall(struct cpumask *dstp) { if (small_const_nbits(small_cpumask_bits)) { cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits); @@ -589,7 +666,7 @@ static inline void cpumask_setall(struct cpumask *dstp) * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask * @dstp: the cpumask pointer */ -static inline void cpumask_clear(struct cpumask *dstp) +static __always_inline void cpumask_clear(struct cpumask *dstp) { bitmap_zero(cpumask_bits(dstp), large_cpumask_bits); } @@ -600,11 +677,11 @@ static inline void cpumask_clear(struct cpumask *dstp) * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns false, else returns true + * Return: false if *@dstp is empty, else returns true */ -static inline bool cpumask_and(struct cpumask *dstp, - const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -616,8 +693,9 @@ static inline bool cpumask_and(struct cpumask *dstp, * @src1p: the first input * @src2p: the second input */ -static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -629,9 +707,9 @@ static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input */ -static inline void cpumask_xor(struct cpumask *dstp, - const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -643,11 +721,11 @@ static inline void cpumask_xor(struct cpumask *dstp, * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns false, else returns true + * Return: false if *@dstp is empty, else returns true */ -static inline bool cpumask_andnot(struct cpumask *dstp, - const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -657,9 +735,11 @@ static inline bool cpumask_andnot(struct cpumask *dstp, * cpumask_equal - *src1p == *src2p * @src1p: the first input * @src2p: the second input + * + * Return: true if the cpumasks are equal, false if not */ -static inline bool cpumask_equal(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -670,10 +750,13 @@ static inline bool cpumask_equal(const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input * @src3p: the third input + * + * Return: true if first cpumask ORed with second cpumask == third cpumask, + * otherwise false */ -static inline bool cpumask_or_equal(const struct cpumask *src1p, - const struct cpumask *src2p, - const struct cpumask *src3p) +static __always_inline +bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p, + const struct cpumask *src3p) { return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p), cpumask_bits(src3p), small_cpumask_bits); @@ -683,9 +766,12 @@ static inline bool cpumask_or_equal(const struct cpumask *src1p, * cpumask_intersects - (*src1p & *src2p) != 0 * @src1p: the first input * @src2p: the second input + * + * Return: true if first cpumask ANDed with second cpumask is non-empty, + * otherwise false */ -static inline bool cpumask_intersects(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -696,10 +782,10 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input * - * Returns true if *@src1p is a subset of *@src2p, else returns false + * Return: true if *@src1p is a subset of *@src2p, else returns false */ -static inline bool cpumask_subset(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -708,8 +794,10 @@ static inline bool cpumask_subset(const struct cpumask *src1p, /** * cpumask_empty - *srcp == 0 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear. + * + * Return: true if srcp is empty (has no bits set), else false */ -static inline bool cpumask_empty(const struct cpumask *srcp) +static __always_inline bool cpumask_empty(const struct cpumask *srcp) { return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits); } @@ -717,8 +805,10 @@ static inline bool cpumask_empty(const struct cpumask *srcp) /** * cpumask_full - *srcp == 0xFFFFFFFF... * @srcp: the cpumask to that all cpus < nr_cpu_ids are set. + * + * Return: true if srcp is full (has all bits set), else false */ -static inline bool cpumask_full(const struct cpumask *srcp) +static __always_inline bool cpumask_full(const struct cpumask *srcp) { return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits); } @@ -726,8 +816,10 @@ static inline bool cpumask_full(const struct cpumask *srcp) /** * cpumask_weight - Count of bits in *srcp * @srcp: the cpumask to count bits (< nr_cpu_ids) in. + * + * Return: count of bits set in *srcp */ -static inline unsigned int cpumask_weight(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp) { return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits); } @@ -736,21 +828,37 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. + * + * Return: count of bits set in both *srcp1 and *srcp2 */ -static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, - const struct cpumask *srcp2) +static __always_inline +unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2) { return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); } +/** + * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2) + * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. + * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. + * + * Return: count of bits set in both *srcp1 and *srcp2 + */ +static __always_inline +unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, + const struct cpumask *srcp2) +{ + return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); +} + /** * cpumask_shift_right - *dstp = *srcp >> n * @dstp: the cpumask result * @srcp: the input to shift * @n: the number of bits to shift by */ -static inline void cpumask_shift_right(struct cpumask *dstp, - const struct cpumask *srcp, int n) +static __always_inline +void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n) { bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n, small_cpumask_bits); @@ -762,8 +870,8 @@ static inline void cpumask_shift_right(struct cpumask *dstp, * @srcp: the input to shift * @n: the number of bits to shift by */ -static inline void cpumask_shift_left(struct cpumask *dstp, - const struct cpumask *srcp, int n) +static __always_inline +void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n) { bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n, nr_cpumask_bits); @@ -774,8 +882,8 @@ static inline void cpumask_shift_left(struct cpumask *dstp, * @dstp: the result * @srcp: the input cpumask */ -static inline void cpumask_copy(struct cpumask *dstp, - const struct cpumask *srcp) +static __always_inline +void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) { bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits); } @@ -784,7 +892,7 @@ static inline void cpumask_copy(struct cpumask *dstp, * cpumask_any - pick a "random" cpu from *srcp * @srcp: the input cpumask * - * Returns >= nr_cpu_ids if no cpus set. + * Return: >= nr_cpu_ids if no cpus set. */ #define cpumask_any(srcp) cpumask_first(srcp) @@ -793,7 +901,7 @@ static inline void cpumask_copy(struct cpumask *dstp, * @mask1: the first input cpumask * @mask2: the second input cpumask * - * Returns >= nr_cpu_ids if no cpus set. + * Return: >= nr_cpu_ids if no cpus set. */ #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2)) @@ -809,10 +917,10 @@ static inline void cpumask_copy(struct cpumask *dstp, * @len: the length of the buffer * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ -static inline int cpumask_parse_user(const char __user *buf, int len, - struct cpumask *dstp) +static __always_inline +int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp) { return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); } @@ -823,10 +931,10 @@ static inline int cpumask_parse_user(const char __user *buf, int len, * @len: the length of the buffer * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ -static inline int cpumask_parselist_user(const char __user *buf, int len, - struct cpumask *dstp) +static __always_inline +int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp) { return bitmap_parselist_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); @@ -837,9 +945,9 @@ static inline int cpumask_parselist_user(const char __user *buf, int len, * @buf: the buffer to extract from * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ -static inline int cpumask_parse(const char *buf, struct cpumask *dstp) +static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp) { return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits); } @@ -849,17 +957,19 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp) * @buf: the buffer to extract from * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ -static inline int cpulist_parse(const char *buf, struct cpumask *dstp) +static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp) { return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits); } /** - * cpumask_size - size to allocate for a 'struct cpumask' in bytes + * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes + * + * Return: size to allocate for a &struct cpumask in bytes */ -static inline unsigned int cpumask_size(void) +static __always_inline unsigned int cpumask_size(void) { return bitmap_size(large_cpumask_bits); } @@ -871,7 +981,7 @@ static inline unsigned int cpumask_size(void) * little more difficult, we typedef cpumask_var_t to an array or a * pointer: doing &mask on an array is a noop, so it still works. * - * ie. + * i.e. * cpumask_var_t tmpmask; * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL)) * return -ENOMEM; @@ -912,7 +1022,7 @@ typedef struct cpumask *cpumask_var_t; bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); -static inline +static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); @@ -927,14 +1037,16 @@ bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) * a nop returning a constant 1 (in ). * * See alloc_cpumask_var_node. + * + * Return: %true if allocation succeeded, %false if not */ -static inline +static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); } -static inline +static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { return alloc_cpumask_var(mask, flags | __GFP_ZERO); @@ -944,7 +1056,7 @@ void alloc_bootmem_cpumask_var(cpumask_var_t *mask); void free_cpumask_var(cpumask_var_t mask); void free_bootmem_cpumask_var(cpumask_var_t mask); -static inline bool cpumask_available(cpumask_var_t mask) +static __always_inline bool cpumask_available(cpumask_var_t mask) { return mask != NULL; } @@ -955,43 +1067,43 @@ typedef struct cpumask cpumask_var_t[1]; #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x) #define __cpumask_var_read_mostly -static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { return true; } -static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, +static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { return true; } -static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { cpumask_clear(*mask); return true; } -static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, +static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { cpumask_clear(*mask); return true; } -static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) +static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) { } -static inline void free_cpumask_var(cpumask_var_t mask) +static __always_inline void free_cpumask_var(cpumask_var_t mask) { } -static inline void free_bootmem_cpumask_var(cpumask_var_t mask) +static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask) { } -static inline bool cpumask_available(cpumask_var_t mask) +static __always_inline bool cpumask_available(cpumask_var_t mask) { return true; } @@ -1037,7 +1149,7 @@ static inline void reset_cpu_possible_mask(void) void set_cpu_online(unsigned int cpu, bool online); /** - * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask * + * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask * * @bitmap: the bitmap * * There are a few places where cpumask_var_t isn't appropriate and @@ -1050,7 +1162,7 @@ void set_cpu_online(unsigned int cpu, bool online); ((struct cpumask *)(1 ? (bitmap) \ : (void *)sizeof(__check_is_bitmap(bitmap)))) -static inline int __check_is_bitmap(const unsigned long *bitmap) +static __always_inline int __check_is_bitmap(const unsigned long *bitmap) { return 1; } @@ -1065,7 +1177,7 @@ static inline int __check_is_bitmap(const unsigned long *bitmap) extern const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; -static inline const struct cpumask *get_cpu_mask(unsigned int cpu) +static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu) { const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; p -= cpu / BITS_PER_LONG; @@ -1080,6 +1192,8 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu) * interface gives only a momentary snapshot and is not protected against * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held * region. + * + * Return: momentary snapshot of the number of online CPUs */ static __always_inline unsigned int num_online_cpus(void) { @@ -1089,27 +1203,27 @@ static __always_inline unsigned int num_online_cpus(void) #define num_present_cpus() cpumask_weight(cpu_present_mask) #define num_active_cpus() cpumask_weight(cpu_active_mask) -static inline bool cpu_online(unsigned int cpu) +static __always_inline bool cpu_online(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_online_mask); } -static inline bool cpu_possible(unsigned int cpu) +static __always_inline bool cpu_possible(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_possible_mask); } -static inline bool cpu_present(unsigned int cpu) +static __always_inline bool cpu_present(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_present_mask); } -static inline bool cpu_active(unsigned int cpu) +static __always_inline bool cpu_active(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_active_mask); } -static inline bool cpu_dying(unsigned int cpu) +static __always_inline bool cpu_dying(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_dying_mask); } @@ -1121,27 +1235,27 @@ static inline bool cpu_dying(unsigned int cpu) #define num_present_cpus() 1U #define num_active_cpus() 1U -static inline bool cpu_online(unsigned int cpu) +static __always_inline bool cpu_online(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_possible(unsigned int cpu) +static __always_inline bool cpu_possible(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_present(unsigned int cpu) +static __always_inline bool cpu_present(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_active(unsigned int cpu) +static __always_inline bool cpu_active(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_dying(unsigned int cpu) +static __always_inline bool cpu_dying(unsigned int cpu) { return false; } @@ -1172,10 +1286,10 @@ static inline bool cpu_dying(unsigned int cpu) * @mask: the cpumask to copy * @buf: the buffer to copy into * - * Returns the length of the (null-terminated) @buf string, zero if + * Return: the length of the (null-terminated) @buf string, zero if * nothing is copied. */ -static inline ssize_t +static __always_inline ssize_t cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) { return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask), @@ -1195,12 +1309,12 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) * cpumask; Typically used by bin_attribute to export cpumask bitmask * ABI. * - * Returns the length of how many bytes have been copied, excluding + * Return: the length of how many bytes have been copied, excluding * terminating '\0'. */ -static inline ssize_t -cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, - loff_t off, size_t count) +static __always_inline +ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, + loff_t off, size_t count) { return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask), nr_cpu_ids, off, count) - 1; @@ -1216,10 +1330,13 @@ cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, * * Everything is same with the above cpumap_print_bitmask_to_buf() * except the print format. + * + * Return: the length of how many bytes have been copied, excluding + * terminating '\0'. */ -static inline ssize_t -cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, - loff_t off, size_t count) +static __always_inline +ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, + loff_t off, size_t count) { return bitmap_print_list_to_buf(buf, cpumask_bits(mask), nr_cpu_ids, off, count) - 1; diff --git a/include/linux/find.h b/include/linux/find.h index a3a908a27df7e9a005cb7d9dbed5e53e3acf2d09..96b8410a7cf76742e0a655e07539a0e06e2ffb7f 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -29,6 +29,8 @@ unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1, const unsign unsigned long n); extern unsigned long _find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size); +unsigned long _find_first_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size); unsigned long _find_first_and_and_bit(const unsigned long *addr1, const unsigned long *addr2, const unsigned long *addr3, unsigned long size); extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); @@ -347,6 +349,29 @@ unsigned long find_first_and_bit(const unsigned long *addr1, } #endif +/** + * find_first_andnot_bit - find the first bit set in 1st memory region and unset in 2nd + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on + * @size: The bitmap size in bits + * + * Returns the bit number for the first set bit + * If no bits are set, returns >= @size. + */ +static __always_inline +unsigned long find_first_andnot_bit(const unsigned long *addr1, + const unsigned long *addr2, + unsigned long size) +{ + if (small_const_nbits(size)) { + unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0); + + return val ? __ffs(val) : size; + } + + return _find_first_andnot_bit(addr1, addr2, size); +} + /** * find_first_and_and_bit - find the first set bit in 3 memory regions * @addr1: The first address to base the search on diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 226cada23934ee1e7709167f81e7d264019a362d..ec690895ee2b057c15718452a106cdec996b09da 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -148,6 +148,11 @@ enum kernfs_root_flag { * Support user xattrs to be written to nodes rooted at this root. */ KERNFS_ROOT_SUPPORT_USER_XATTR = 0x0008, + + /* + * Renames must not change the parent node. + */ + KERNFS_ROOT_INVARIANT_PARENT = 0x0010, }; /* type-specific structures for kernfs_node union members */ @@ -202,8 +207,8 @@ struct kernfs_node { * never moved to a different parent, it is safe to access the * parent directly. */ - struct kernfs_node *parent; - const char *name; + struct kernfs_node __rcu *__parent; + const char __rcu *name; struct rb_node rb; @@ -326,6 +331,7 @@ struct kernfs_ops { struct poll_table_struct *pt); int (*mmap)(struct kernfs_open_file *of, struct vm_area_struct *vma); + loff_t (*llseek)(struct kernfs_open_file *of, loff_t offset, int whence); CK_KABI_RESERVE(1) CK_KABI_RESERVE(2) @@ -405,7 +411,7 @@ static inline bool kernfs_ns_enabled(struct kernfs_node *kn) } int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen); -int kernfs_path_from_node(struct kernfs_node *root_kn, struct kernfs_node *kn, +int kernfs_path_from_node(struct kernfs_node *kn_to, struct kernfs_node *kn_from, char *buf, size_t buflen); void pr_cont_kernfs_name(struct kernfs_node *kn); void pr_cont_kernfs_path(struct kernfs_node *kn); @@ -426,6 +432,7 @@ struct dentry *kernfs_node_dentry(struct kernfs_node *kn, struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops, unsigned int flags, void *priv); void kernfs_destroy_root(struct kernfs_root *root); +unsigned int kernfs_root_flags(struct kernfs_node *kn); struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent, const char *name, umode_t mode, @@ -524,6 +531,8 @@ kernfs_create_root(struct kernfs_syscall_ops *scops, unsigned int flags, { return ERR_PTR(-ENOSYS); } static inline void kernfs_destroy_root(struct kernfs_root *root) { } +static inline unsigned int kernfs_root_flags(struct kernfs_node *kn) +{ return 0; } static inline struct kernfs_node * kernfs_create_dir_ns(struct kernfs_node *parent, const char *name, diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 01bef03c0eddd4af484fe79e54a53931e2699784..6fb4894b8cfd1fb61d12702867636809edd3eb4c 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -2,7 +2,7 @@ #ifndef _RESCTRL_H #define _RESCTRL_H -#include +#include #include #include #include @@ -12,8 +12,6 @@ #include #endif -extern struct mutex rdtgroup_mutex; - /* CLOSID, RMID value used by the default control group */ #define RESCTRL_RESERVED_CLOSID 0 #define RESCTRL_RESERVED_RMID 0 @@ -32,16 +30,49 @@ int proc_resctrl_show(struct seq_file *m, /* max value for struct rdt_domain's mbps_val */ #define MBA_MAX_MBPS U32_MAX -/* - * Resctrl uses u32 to hold the user-space config. The maximum bitmap size is - * 32. - */ -#define RESCTRL_MAX_CBM 32 +/* Walk all possible resources, with variants for only controls or monitors. */ +#define for_each_rdt_resource(_r) \ + for ((_r) = resctrl_arch_get_resource(0); \ + (_r) && (_r)->rid < RDT_NUM_RESOURCES; \ + (_r) = resctrl_arch_get_resource((_r)->rid + 1)) -extern unsigned int resctrl_rmid_realloc_limit; -extern unsigned int resctrl_rmid_realloc_threshold; +#define for_each_capable_rdt_resource(r) \ + for_each_rdt_resource((r)) \ + if ((r)->alloc_capable || (r)->mon_capable) + +#define for_each_alloc_capable_rdt_resource(r) \ + for_each_rdt_resource((r)) \ + if ((r)->alloc_capable) + +#define for_each_mon_capable_rdt_resource(r) \ + for_each_rdt_resource((r)) \ + if ((r)->mon_capable) + +enum resctrl_res_level { + RDT_RESOURCE_L3, + RDT_RESOURCE_L2, + RDT_RESOURCE_MBA, + RDT_RESOURCE_SMBA, + + /* Must be the last */ + RDT_NUM_RESOURCES, +}; /** + * enum resctrl_conf_type - The type of configuration. + * @CDP_NONE: No prioritisation, both code and data are controlled or monitored. + * @CDP_CODE: Configuration applies to instruction fetches. + * @CDP_DATA: Configuration applies to reads and writes. + */ +enum resctrl_conf_type { + CDP_NONE, + CDP_CODE, + CDP_DATA, +}; + +#define CDP_NUM_TYPES (CDP_DATA + 1) + +/* * struct pseudo_lock_region - pseudo-lock region information * @s: Resctrl schema for the resource to which this * pseudo-locked region belongs @@ -67,7 +98,7 @@ extern unsigned int resctrl_rmid_realloc_threshold; struct pseudo_lock_region { struct resctrl_schema *s; u32 closid; - struct rdt_domain *d; + struct rdt_ctrl_domain *d; u32 cbm; wait_queue_head_t lock_thread_wq; int thread_done; @@ -90,11 +121,45 @@ struct resctrl_staged_config { bool have_new_ctrl; }; +enum resctrl_domain_type { + RESCTRL_CTRL_DOMAIN, + RESCTRL_MON_DOMAIN, +}; + /** - * struct rdt_domain - group of CPUs sharing a resctrl resource + * struct rdt_domain_hdr - common header for different domain types * @list: all instances of this resource * @id: unique id for this instance + * @type: type of this instance * @cpu_mask: which CPUs share this resource + */ +struct rdt_domain_hdr { + struct list_head list; + int id; + enum resctrl_domain_type type; + struct cpumask cpu_mask; +}; + +/** + * struct rdt_ctrl_domain - group of CPUs sharing a resctrl control resource + * @hdr: common header for different domain types + * @plr: pseudo-locked region (if any) associated with domain + * @staged_config: parsed configuration to be applied + * @mbps_val: When mba_sc is enabled, this holds the array of user + * specified control values for mba_sc in MBps, indexed + * by closid + */ +struct rdt_ctrl_domain { + struct rdt_domain_hdr hdr; + struct pseudo_lock_region *plr; + struct resctrl_staged_config staged_config[CDP_NUM_TYPES]; + u32 *mbps_val; +}; + +/** + * struct rdt_mon_domain - group of CPUs sharing a resctrl monitor resource + * @hdr: common header for different domain types + * @ci_id: cache info id for this domain * @rmid_busy_llc: bitmap of which limbo RMIDs are above threshold * @mbm_total: saved state for MBM total bandwidth * @mbm_local: saved state for MBM local bandwidth @@ -102,17 +167,10 @@ struct resctrl_staged_config { * @cqm_limbo: worker to periodically read CQM h/w counters * @mbm_work_cpu: worker CPU for MBM h/w counters * @cqm_work_cpu: worker CPU for CQM h/w counters - * @mbm_cntr_map: bitmap to track domain counter assignment - * @plr: pseudo-locked region (if any) associated with domain - * @staged_config: parsed configuration to be applied - * @mbps_val: When mba_sc is enabled, this holds the array of user - * specified control values for mba_sc in MBps, indexed - * by closid */ -struct rdt_domain { - struct list_head list; - int id; - struct cpumask cpu_mask; +struct rdt_mon_domain { + struct rdt_domain_hdr hdr; + unsigned int ci_id; unsigned long *rmid_busy_llc; struct mbm_state *mbm_total; struct mbm_state *mbm_local; @@ -120,10 +178,6 @@ struct rdt_domain { struct delayed_work cqm_limbo; int mbm_work_cpu; int cqm_work_cpu; - unsigned long *mbm_cntr_map; - struct pseudo_lock_region *plr; - struct resctrl_staged_config staged_config[CDP_NUM_TYPES]; - u32 *mbps_val; }; /** @@ -137,8 +191,6 @@ struct rdt_domain { * @arch_has_sparse_bitmasks: True if a bitmask like f00f is valid. * @arch_has_per_cpu_cfg: True if QOS_CFG register for this cache * level has CPU scope. - * @io_alloc_capable: True if portion of the cache can be configured - * for I/O traffic. */ struct resctrl_cache { unsigned int cbm_len; @@ -146,7 +198,6 @@ struct resctrl_cache { unsigned int shareable_bits; bool arch_has_sparse_bitmasks; bool arch_has_per_cpu_cfg; - bool io_alloc_capable; }; /** @@ -166,38 +217,42 @@ enum membw_throttle_mode { /** * struct resctrl_membw - Memory bandwidth allocation related data * @min_bw: Minimum memory bandwidth percentage user can request + * @max_bw: Maximum memory bandwidth value, used as the reset value * @bw_gran: Granularity at which the memory bandwidth is allocated * @delay_linear: True if memory B/W delay is in linear scale * @arch_needs_linear: True if we can't configure non-linear resources * @throttle_mode: Bandwidth throttling mode when threads request * different memory bandwidths * @mba_sc: True if MBA software controller(mba_sc) is enabled - * @hwdrc_mb: True if memory bandwidth HWDRC is enabled * @mb_map: Mapping of memory B/W percentage to memory B/W delay */ struct resctrl_membw { u32 min_bw; + u32 max_bw; u32 bw_gran; u32 delay_linear; bool arch_needs_linear; enum membw_throttle_mode throttle_mode; bool mba_sc; - bool hwdrc_mb; u32 *mb_map; }; +struct resctrl_schema; + +enum resctrl_scope { + RESCTRL_L2_CACHE = 2, + RESCTRL_L3_CACHE = 3, + RESCTRL_L3_NODE, +}; + /** - * struct resctrl_mon - Monitoring related data - * @num_rmid: Number of RMIDs available - * @num_mbm_cntrs: Number of monitoring counters - * @mbm_cntr_assignable:Is system capable of supporting monitor assignment? - * @evt_list: List of monitoring events + * enum resctrl_schema_fmt - The format user-space provides for a schema. + * @RESCTRL_SCHEMA_BITMAP: The schema is a bitmap in hex. + * @RESCTRL_SCHEMA_RANGE: The schema is a decimal number. */ -struct resctrl_mon { - int num_rmid; - int num_mbm_cntrs; - bool mbm_cntr_assignable; - struct list_head evt_list; +enum resctrl_schema_fmt { + RESCTRL_SCHEMA_BITMAP, + RESCTRL_SCHEMA_RANGE, }; /** @@ -205,36 +260,36 @@ struct resctrl_mon { * @rid: The index of the resource * @alloc_capable: Is allocation available on this machine * @mon_capable: Is monitor feature available on this machine - * @cache_level: Which cache level defines scope of this resource + * @num_rmid: Number of RMIDs available + * @ctrl_scope: Scope of this resource for control functions + * @mon_scope: Scope of this resource for monitor functions * @cache: Cache allocation related data * @membw: If the component has bandwidth controls, their properties. - * @domains: RCU list of all domains for this resource + * @ctrl_domains: RCU list of all control domains for this resource + * @mon_domains: RCU list of all monitor domains for this resource * @name: Name to use in "schemata" file. - * @data_width: Character width of data when displaying - * @default_ctrl: Specifies default cache cbm or memory B/W percent. - * @format_str: Per resource format string to show domain value - * @fflags: flags to choose base and info files - * @mbm_cfg_mask: Bandwidth sources that can be tracked when Bandwidth - * Monitoring Event Configuration (BMEC) is supported. + * @schema_fmt: Which format string and parser is used for this schema. + * @evt_list: List of monitoring events + * @mbm_cfg_mask: Bandwidth sources that can be tracked when bandwidth + * monitoring events can be configured. * @cdp_capable: Is the CDP feature available on this resource */ struct rdt_resource { int rid; bool alloc_capable; bool mon_capable; - int cache_level; + int num_rmid; + enum resctrl_scope ctrl_scope; + enum resctrl_scope mon_scope; struct resctrl_cache cache; struct resctrl_membw membw; - struct resctrl_mon mon; - struct list_head domains; + struct list_head ctrl_domains; + struct list_head mon_domains; char *name; - int data_width; - u32 default_ctrl; - const char *format_str; - unsigned long fflags; + enum resctrl_schema_fmt schema_fmt; + struct list_head evt_list; unsigned int mbm_cfg_mask; bool cdp_capable; - struct rdt_domain *rdt_domain_list[NR_CPUS]; }; /* @@ -249,6 +304,7 @@ struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l); * user-space * @list: Member of resctrl_schema_all. * @name: The name to use in the "schemata" file. + * @fmt_str: Format string to show domain value. * @conf_type: Whether this schema is specific to code/data. * @res: The resource structure exported by the architecture to describe * the hardware that is configured by this schema. @@ -259,67 +315,89 @@ struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l); struct resctrl_schema { struct list_head list; char name[8]; + const char *fmt_str; enum resctrl_conf_type conf_type; struct rdt_resource *res; u32 num_closid; }; -struct resctrl_cpu_sync { +struct resctrl_cpu_defaults { u32 closid; u32 rmid; }; struct resctrl_mon_config_info { - struct rdt_resource *r; - struct rdt_domain *d; - u32 evtid; - u32 mon_config; - int err; + struct rdt_resource *r; + struct rdt_mon_domain *d; + u32 evtid; + u32 mon_config; }; -/* - * Assignment flags for ABMC feature +/** + * resctrl_arch_sync_cpu_closid_rmid() - Refresh this CPU's CLOSID and RMID. + * Call via IPI. + * @info: If non-NULL, a pointer to a struct resctrl_cpu_defaults + * specifying the new CLOSID and RMID for tasks in the default + * resctrl ctrl and mon group when running on this CPU. If NULL, + * this CPU is not re-assigned to a different default group. + * + * Propagates reassignment of CPUs and/or tasks to different resctrl groups + * when requested by the resctrl core code. + * + * This function records the per-cpu defaults specified by @info (if any), + * and then reconfigures the CPU's hardware CLOSID and RMID for subsequent + * execution based on @current, in the same way as during a task switch. */ -#define ASSIGN_NONE 0 -#define ASSIGN_TOTAL BIT(QOS_L3_MBM_TOTAL_EVENT_ID) -#define ASSIGN_LOCAL BIT(QOS_L3_MBM_LOCAL_EVENT_ID) +void resctrl_arch_sync_cpu_closid_rmid(void *info); /** - * mon_event_config_index_get - get the hardware index for the - * configurable event - * @evtid: event id. - * - * Return: 0 for evtid == QOS_L3_MBM_TOTAL_EVENT_ID - * 1 for evtid == QOS_L3_MBM_LOCAL_EVENT_ID - * INVALID_CONFIG_INDEX for invalid evtid + * resctrl_get_default_ctrl() - Return the default control value for this + * resource. + * @r: The resource whose default control type is queried. */ -static inline unsigned int mon_event_config_index_get(u32 evtid) +static inline u32 resctrl_get_default_ctrl(struct rdt_resource *r) { - switch (evtid) { - case QOS_L3_MBM_TOTAL_EVENT_ID: - return 0; - case QOS_L3_MBM_LOCAL_EVENT_ID: - return 1; - default: - /* Should never reach here */ - return INVALID_CONFIG_INDEX; + switch (r->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + return BIT_MASK(r->cache.cbm_len) - 1; + case RESCTRL_SCHEMA_RANGE: + return r->membw.max_bw; } -} -/* - * Update and re-load this CPUs defaults. Called via IPI, takes a pointer to - * struct resctrl_cpu_sync, or NULL. - */ -void resctrl_arch_sync_cpu_defaults(void *info); + return WARN_ON_ONCE(1); +} /* The number of closid supported by this resource regardless of CDP */ u32 resctrl_arch_get_num_closid(struct rdt_resource *r); - -struct rdt_domain *resctrl_arch_find_domain(struct rdt_resource *r, int id); +u32 resctrl_arch_system_num_rmid_idx(void); int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid); bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt); +/** + * resctrl_arch_mon_event_config_write() - Write the config for an event. + * @config_info: struct resctrl_mon_config_info describing the resource, domain + * and event. + * + * Reads resource, domain and eventid from @config_info and writes the + * event config_info->mon_config into hardware. + * + * Called via IPI to reach a CPU that is a member of the specified domain. + */ +void resctrl_arch_mon_event_config_write(void *config_info); + +/** + * resctrl_arch_mon_event_config_read() - Read the config for an event. + * @config_info: struct resctrl_mon_config_info describing the resource, domain + * and event. + * + * Reads resource, domain and eventid from @config_info and reads the + * hardware config value into config_info->mon_config. + * + * Called via IPI to reach a CPU that is a member of the specified domain. + */ +void resctrl_arch_mon_event_config_read(void *config_info); + /* For use by arch code to remap resctrl's smaller CDP CLOSID range */ static inline u32 resctrl_get_config_index(u32 closid, enum resctrl_conf_type type) @@ -329,41 +407,28 @@ static inline u32 resctrl_get_config_index(u32 closid, case CDP_NONE: return closid; case CDP_CODE: - return (closid * 2) + 1; + return closid * 2 + 1; case CDP_DATA: - return (closid * 2); + return closid * 2; } } -/* - * Caller must hold the cpuhp read lock to prevent the struct rdt_domain being - * freed. - */ -static inline struct rdt_domain * -resctrl_get_domain_from_cpu(int cpu, struct rdt_resource *r) -{ - struct rdt_domain *d; - - list_for_each_entry_rcu(d, &r->domains, list) { - /* Find the domain that contains this CPU */ - if (cpumask_test_cpu(cpu, &d->cpu_mask)) - return d; - } - - return NULL; -} +bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level l); +int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable); /* * Update the ctrl_val and apply this config right now. * Must be called on one of the domain's CPUs. */ -int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, +int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type t, u32 cfg_val); -u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d, +u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type type); -int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d); -void resctrl_offline_domain(struct rdt_resource *r, struct rdt_domain *d); +int resctrl_online_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d); +int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d); +void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d); +void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d); void resctrl_online_cpu(unsigned int cpu); void resctrl_offline_cpu(unsigned int cpu); @@ -392,7 +457,7 @@ void resctrl_offline_cpu(unsigned int cpu); * Return: * 0 on success, or -EIO, -EINVAL etc on error. */ -int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id eventid, u64 *val, void *arch_mon_ctx); @@ -413,6 +478,20 @@ static inline void resctrl_arch_rmid_read_context_check(void) might_sleep(); } +/** + * resctrl_find_domain() - Search for a domain id in a resource domain list. + * @h: The domain list to search. + * @id: The domain id to search for. + * @pos: A pointer to position in the list id should be inserted. + * + * Search the domain list to find the domain id. If the domain id is + * found, return the domain. NULL otherwise. If the domain id is not + * found (and NULL returned) then the first domain with id bigger than + * the input id can be returned to the caller via @pos. + */ +struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id, + struct list_head **pos); + /** * resctrl_arch_reset_rmid() - Reset any private state associated with rmid * and eventid. @@ -425,7 +504,7 @@ static inline void resctrl_arch_rmid_read_context_check(void) * * This can be called from any CPU. */ -void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id eventid); @@ -438,34 +517,34 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, * * This can be called from any CPU. */ -void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_domain *d); +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain *d); /** - * resctrl_arch_io_alloc_enable() - Enable/disable io_alloc feature. - * @r: The resctrl resource. - * @enable: Enable (true) or disable (false) io_alloc on resource @r. + * resctrl_arch_reset_all_ctrls() - Reset the control for each CLOSID to its + * default. + * @r: The resctrl resource to reset. * * This can be called from any CPU. - * - * Return: - * 0 on success, <0 on error. - */ -int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable); - -/** - * resctrl_arch_get_io_alloc_enabled() - Get io_alloc feature state. - * @r: The resctrl resource. - * - * Return: - * true if io_alloc is enabled or false if disabled. */ -bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r); +void resctrl_arch_reset_all_ctrls(struct rdt_resource *r); extern unsigned int resctrl_rmid_realloc_threshold; extern unsigned int resctrl_rmid_realloc_limit; -void resctrl_file_fflags_init(const char *config, unsigned long fflags); int resctrl_init(void); void resctrl_exit(void); +#ifdef CONFIG_RESCTRL_FS_PSEUDO_LOCK +u64 resctrl_arch_get_prefetch_disable_bits(void); +int resctrl_arch_pseudo_lock_fn(void *_plr); +int resctrl_arch_measure_cycles_lat_fn(void *_plr); +int resctrl_arch_measure_l2_residency(void *_plr); +int resctrl_arch_measure_l3_residency(void *_plr); +#else +static inline u64 resctrl_arch_get_prefetch_disable_bits(void) { return 0; } +static inline int resctrl_arch_pseudo_lock_fn(void *_plr) { return 0; } +static inline int resctrl_arch_measure_cycles_lat_fn(void *_plr) { return 0; } +static inline int resctrl_arch_measure_l2_residency(void *_plr) { return 0; } +static inline int resctrl_arch_measure_l3_residency(void *_plr) { return 0; } +#endif /* CONFIG_RESCTRL_FS_PSEUDO_LOCK */ #endif /* _RESCTRL_H */ diff --git a/include/linux/resctrl_types.h b/include/linux/resctrl_types.h index 7c6dad3df4ff9baa0e3f03cc38f3803bc016ee04..a25fb9c4070d3cc509298df519f7078cd124b4cd 100644 --- a/include/linux/resctrl_types.h +++ b/include/linux/resctrl_types.h @@ -1,71 +1,38 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (C) 2024 Arm Ltd. + * Copyright (C) 2025 Arm Ltd. * Based on arch/x86/kernel/cpu/resctrl/internal.h */ #ifndef __LINUX_RESCTRL_TYPES_H #define __LINUX_RESCTRL_TYPES_H -#define INVALID_CONFIG_VALUE U32_MAX -#define INVALID_CONFIG_INDEX UINT_MAX - -#define CQM_LIMBOCHECK_INTERVAL 1000 - -#define MBM_CNTR_WIDTH_BASE 24 -#define MBM_OVERFLOW_INTERVAL 1000 #define MAX_MBA_BW 100u -#define MBA_IS_LINEAR 0x4 +#define MBM_OVERFLOW_INTERVAL 1000 -/* rdtgroup.flags */ -#define RDT_DELETED 1 +/* Reads to Local DRAM Memory */ +#define READS_TO_LOCAL_MEM BIT(0) -/* rftype.flags */ -#define RFTYPE_FLAGS_CPUS_LIST 1 +/* Reads to Remote DRAM Memory */ +#define READS_TO_REMOTE_MEM BIT(1) -/* - * Define the file type flags for base and info directories. - */ -#define RFTYPE_INFO BIT(0) -#define RFTYPE_BASE BIT(1) -#define RFTYPE_CTRL BIT(4) -#define RFTYPE_MON BIT(5) -#define RFTYPE_TOP BIT(6) -#define RFTYPE_RES_CACHE BIT(8) -#define RFTYPE_RES_MB BIT(9) -#define RFTYPE_DEBUG BIT(10) -#define RFTYPE_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL) -#define RFTYPE_MON_INFO (RFTYPE_INFO | RFTYPE_MON) -#define RFTYPE_TOP_INFO (RFTYPE_INFO | RFTYPE_TOP) -#define RFTYPE_CTRL_BASE (RFTYPE_BASE | RFTYPE_CTRL) -#define RFTYPE_MON_BASE (RFTYPE_BASE | RFTYPE_MON) +/* Non-Temporal Writes to Local Memory */ +#define NON_TEMP_WRITE_TO_LOCAL_MEM BIT(2) -/* Max event bits supported */ -#define MAX_EVT_CONFIG_BITS GENMASK(6, 0) +/* Non-Temporal Writes to Remote Memory */ +#define NON_TEMP_WRITE_TO_REMOTE_MEM BIT(3) -/** - * enum resctrl_conf_type - The type of configuration. - * @CDP_NONE: No prioritisation, both code and data are controlled or monitored. - * @CDP_CODE: Configuration applies to instruction fetches. - * @CDP_DATA: Configuration applies to reads and writes. - */ -enum resctrl_conf_type { - CDP_NONE, - CDP_CODE, - CDP_DATA, -}; +/* Reads to Local Memory the system identifies as "Slow Memory" */ +#define READS_TO_LOCAL_S_MEM BIT(4) -enum resctrl_res_level { - RDT_RESOURCE_L3, - RDT_RESOURCE_L2, - RDT_RESOURCE_MBA, - RDT_RESOURCE_SMBA, +/* Reads to Remote Memory the system identifies as "Slow Memory" */ +#define READS_TO_REMOTE_S_MEM BIT(5) - /* Must be the last */ - RDT_NUM_RESOURCES, -}; +/* Dirty Victims to All Types of Memory */ +#define DIRTY_VICTIMS_TO_ALL_MEM BIT(6) -#define CDP_NUM_TYPES (CDP_DATA + 1) +/* Max event bits supported */ +#define MAX_EVT_CONFIG_BITS GENMASK(6, 0) /* * Event IDs, the values match those used to program IA32_QM_EVTSEL before @@ -75,9 +42,9 @@ enum resctrl_event_id { QOS_L3_OCCUP_EVENT_ID = 0x01, QOS_L3_MBM_TOTAL_EVENT_ID = 0x02, QOS_L3_MBM_LOCAL_EVENT_ID = 0x03, - QOS_MC_MBM_BPS_EVENT_ID = 0x04, -}; -#define RESCTRL_MAX_EVENT_NUM 4 + /* Must be the last */ + QOS_NUM_EVENTS, +}; #endif /* __LINUX_RESCTRL_TYPES_H */ diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index fbb7915526b4d5d8df276dd2e49802a04680a607..a40cfd455fe5fe891c777c909f2a7b3309e6ee5c 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -233,4 +233,6 @@ static inline void task_unlock(struct task_struct *p) spin_unlock(&p->alloc_lock); } +DEFINE_GUARD(task_lock, struct task_struct *, task_lock(_T), task_unlock(_T)) + #endif /* _LINUX_SCHED_TASK_H */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index ceb56b39c70f775a736ad2918a2d0901af240f40..90bc853cafb6aeedd433d3016f17a86086df817e 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -548,5 +548,31 @@ DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t, DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try, spin_trylock_irqsave(_T->lock, _T->flags)) +DEFINE_LOCK_GUARD_1(read_lock, rwlock_t, + read_lock(_T->lock), + read_unlock(_T->lock)) + +DEFINE_LOCK_GUARD_1(read_lock_irq, rwlock_t, + read_lock_irq(_T->lock), + read_unlock_irq(_T->lock)) + +DEFINE_LOCK_GUARD_1(read_lock_irqsave, rwlock_t, + read_lock_irqsave(_T->lock, _T->flags), + read_unlock_irqrestore(_T->lock, _T->flags), + unsigned long flags) + +DEFINE_LOCK_GUARD_1(write_lock, rwlock_t, + write_lock(_T->lock), + write_unlock(_T->lock)) + +DEFINE_LOCK_GUARD_1(write_lock_irq, rwlock_t, + write_lock_irq(_T->lock), + write_unlock_irq(_T->lock)) + +DEFINE_LOCK_GUARD_1(write_lock_irqsave, rwlock_t, + write_lock_irqsave(_T->lock, _T->flags), + write_unlock_irqrestore(_T->lock, _T->flags), + unsigned long flags) + #undef __LINUX_INSIDE_SPINLOCK_H #endif /* __LINUX_SPINLOCK_H */ diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h index e9e5e4de9b31221a615657b69e7d3d76b14be253..94627c2512b152218dae853334f144e365c83840 100644 --- a/include/linux/sysfs.h +++ b/include/linux/sysfs.h @@ -227,6 +227,8 @@ struct bin_attribute { char *, loff_t, size_t); ssize_t (*write_new)(struct file *, struct kobject *, const struct bin_attribute *, char *, loff_t, size_t); + loff_t (*llseek)(struct file *, struct kobject *, struct bin_attribute *, + loff_t, int); int (*mmap)(struct file *, struct kobject *, struct bin_attribute *attr, struct vm_area_struct *vma); diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 85be02fde19b431c41992cf69da6a92030250f50..9d4a477bbd37ca1b717149026a3c7a46f7b2ca52 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -223,6 +223,7 @@ static inline void get_css_set(struct css_set *cset) bool cgroup_ssid_enabled(int ssid); +struct cgroup *kn_priv(struct kernfs_node *kn); struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root); struct cgroup *task_cgroup_from_root(struct task_struct *task, struct cgroup_root *root); diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index 8be798cd9896a5b8f081aaeca837724facb1eeb7..cbe0ae63db37352fe9b09e0d0eca74a8ccea347a 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -635,7 +635,7 @@ static ssize_t cgroup_pool_size_write(struct kernfs_open_file *of, u64 val; int i; - cgrp = of->kn->parent->priv; + cgrp = kn_priv(of->kn); if (kstrtoull(buf, 0, &val)) return -EINVAL; @@ -942,7 +942,7 @@ static int cgroup1_rename(struct kernfs_node *kn, struct kernfs_node *new_parent if (kernfs_type(kn) != KERNFS_DIR) return -ENOTDIR; - if (kn->parent != new_parent) + if (rcu_access_pointer(kn->__parent) != new_parent) return -EIO; /* diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 0a0a11f88bd4799b5abd7f60310d42fb42dc7514..9b229efacaa5055294e4ac558ce416e46e3b1cea 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -646,9 +646,22 @@ int cgroup_task_count(const struct cgroup *cgrp) return count; } +struct cgroup *kn_priv(struct kernfs_node *kn) +{ + struct kernfs_node *parent; + /* + * The parent can not be replaced due to KERNFS_ROOT_INVARIANT_PARENT. + * Therefore it is always safe to dereference this pointer outside of a + * RCU section. + */ + parent = rcu_dereference_check(kn->__parent, + kernfs_root_flags(kn) & KERNFS_ROOT_INVARIANT_PARENT); + return parent->priv; +} + struct cgroup_subsys_state *of_css(struct kernfs_open_file *of) { - struct cgroup *cgrp = of->kn->parent->priv; + struct cgroup *cgrp = kn_priv(of->kn); struct cftype *cft = of_cft(of); /* @@ -1635,7 +1648,7 @@ void cgroup_kn_unlock(struct kernfs_node *kn) if (kernfs_type(kn) == KERNFS_DIR) cgrp = kn->priv; else - cgrp = kn->parent->priv; + cgrp = kn_priv(kn); cgroup_unlock(); @@ -1667,7 +1680,7 @@ struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) if (kernfs_type(kn) == KERNFS_DIR) cgrp = kn->priv; else - cgrp = kn->parent->priv; + cgrp = kn_priv(kn); /* * We're gonna grab cgroup_mutex which nests outside kernfs @@ -2109,7 +2122,8 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) root->kf_root = kernfs_create_root(kf_sops, KERNFS_ROOT_CREATE_DEACTIVATED | KERNFS_ROOT_SUPPORT_EXPORTOP | - KERNFS_ROOT_SUPPORT_USER_XATTR, + KERNFS_ROOT_SUPPORT_USER_XATTR | + KERNFS_ROOT_INVARIANT_PARENT, root_cgrp); if (IS_ERR(root->kf_root)) { ret = PTR_ERR(root->kf_root); @@ -4175,7 +4189,7 @@ static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { struct cgroup_file_ctx *ctx = of->priv; - struct cgroup *cgrp = of->kn->parent->priv; + struct cgroup *cgrp = kn_priv(of->kn); struct cftype *cft = of_cft(of); struct cgroup_subsys_state *css; int ret; diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 443057bee87cb7469990f3ad1d6abe09a42c22f3..f63c6b5a18f068f6bd8131ca2dba77015abbc332 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -386,6 +386,34 @@ static int check_ptrace_options(unsigned long data) return 0; } +static inline void ptrace_set_stopped(struct task_struct *task) +{ + guard(spinlock)(&task->sighand->siglock); + + /* + * If the task is already STOPPED, set JOBCTL_TRAP_STOP and + * TRAPPING, and kick it so that it transits to TRACED. TRAPPING + * will be cleared if the child completes the transition or any + * event which clears the group stop states happens. We'll wait + * for the transition to complete before returning from this + * function. + * + * This hides STOPPED -> RUNNING -> TRACED transition from the + * attaching thread but a different thread in the same group can + * still observe the transient RUNNING state. IOW, if another + * thread's WNOHANG wait(2) on the stopped tracee races against + * ATTACH, the wait(2) may fail due to the transient RUNNING. + * + * The following task_is_stopped() test is safe as both transitions + * in and out of STOPPED are protected by siglock. + */ + if (task_is_stopped(task) && + task_set_jobctl_pending(task, JOBCTL_TRAP_STOP | JOBCTL_TRAPPING)) { + task->jobctl &= ~JOBCTL_STOPPED; + signal_wake_up_state(task, __TASK_STOPPED); + } +} + static int ptrace_attach(struct task_struct *task, long request, unsigned long addr, unsigned long flags) @@ -393,17 +421,17 @@ static int ptrace_attach(struct task_struct *task, long request, bool seize = (request == PTRACE_SEIZE); int retval; - retval = -EIO; if (seize) { if (addr != 0) - goto out; + return -EIO; /* * This duplicates the check in check_ptrace_options() because * ptrace_attach() and ptrace_setoptions() have historically * used different error codes for unknown ptrace options. */ if (flags & ~(unsigned long)PTRACE_O_MASK) - goto out; + return -EIO; + retval = check_ptrace_options(flags); if (retval) return retval; @@ -414,88 +442,54 @@ static int ptrace_attach(struct task_struct *task, long request, audit_ptrace(task); - retval = -EPERM; if (unlikely(task->flags & PF_KTHREAD)) - goto out; + return -EPERM; if (same_thread_group(task, current)) - goto out; + return -EPERM; /* * Protect exec's credential calculations against our interference; * SUID, SGID and LSM creds get determined differently * under ptrace. */ - retval = -ERESTARTNOINTR; - if (mutex_lock_interruptible(&task->signal->cred_guard_mutex)) - goto out; + scoped_cond_guard (mutex_intr, return -ERESTARTNOINTR, + &task->signal->cred_guard_mutex) { - task_lock(task); - retval = __ptrace_may_access(task, PTRACE_MODE_ATTACH_REALCREDS); - task_unlock(task); - if (retval) - goto unlock_creds; + scoped_guard (task_lock, task) { + retval = __ptrace_may_access(task, PTRACE_MODE_ATTACH_REALCREDS); + if (retval) + return retval; + } - write_lock_irq(&tasklist_lock); - retval = -EPERM; - if (unlikely(task->exit_state)) - goto unlock_tasklist; - if (task->ptrace) - goto unlock_tasklist; + scoped_guard (write_lock_irq, &tasklist_lock) { + if (unlikely(task->exit_state)) + return -EPERM; + if (task->ptrace) + return -EPERM; - task->ptrace = flags; + task->ptrace = flags; - ptrace_link(task, current); + ptrace_link(task, current); - /* SEIZE doesn't trap tracee on attach */ - if (!seize) - send_sig_info(SIGSTOP, SEND_SIG_PRIV, task); + /* SEIZE doesn't trap tracee on attach */ + if (!seize) + send_sig_info(SIGSTOP, SEND_SIG_PRIV, task); - spin_lock(&task->sighand->siglock); + ptrace_set_stopped(task); + } + } /* - * If the task is already STOPPED, set JOBCTL_TRAP_STOP and - * TRAPPING, and kick it so that it transits to TRACED. TRAPPING - * will be cleared if the child completes the transition or any - * event which clears the group stop states happens. We'll wait - * for the transition to complete before returning from this - * function. - * - * This hides STOPPED -> RUNNING -> TRACED transition from the - * attaching thread but a different thread in the same group can - * still observe the transient RUNNING state. IOW, if another - * thread's WNOHANG wait(2) on the stopped tracee races against - * ATTACH, the wait(2) may fail due to the transient RUNNING. - * - * The following task_is_stopped() test is safe as both transitions - * in and out of STOPPED are protected by siglock. + * We do not bother to change retval or clear JOBCTL_TRAPPING + * if wait_on_bit() was interrupted by SIGKILL. The tracer will + * not return to user-mode, it will exit and clear this bit in + * __ptrace_unlink() if it wasn't already cleared by the tracee; + * and until then nobody can ptrace this task. */ - if (task_is_stopped(task) && - task_set_jobctl_pending(task, JOBCTL_TRAP_STOP | JOBCTL_TRAPPING)) { - task->jobctl &= ~JOBCTL_STOPPED; - signal_wake_up_state(task, __TASK_STOPPED); - } - - spin_unlock(&task->sighand->siglock); - - retval = 0; -unlock_tasklist: - write_unlock_irq(&tasklist_lock); -unlock_creds: - mutex_unlock(&task->signal->cred_guard_mutex); -out: - if (!retval) { - /* - * We do not bother to change retval or clear JOBCTL_TRAPPING - * if wait_on_bit() was interrupted by SIGKILL. The tracer will - * not return to user-mode, it will exit and clear this bit in - * __ptrace_unlink() if it wasn't already cleared by the tracee; - * and until then nobody can ptrace this task. - */ - wait_on_bit(&task->jobctl, JOBCTL_TRAPPING_BIT, TASK_KILLABLE); - proc_ptrace_connector(task, PTRACE_ATTACH); - } + wait_on_bit(&task->jobctl, JOBCTL_TRAPPING_BIT, TASK_KILLABLE); + proc_ptrace_connector(task, PTRACE_ATTACH); - return retval; + return 0; } /** diff --git a/kernel/sched/group_balancer.c b/kernel/sched/group_balancer.c index de46b2a0d7588844716a42b18d42e6902ed88807..3a5839f21eff4f100978f14d3fb20f8dd6cd9b84 100644 --- a/kernel/sched/group_balancer.c +++ b/kernel/sched/group_balancer.c @@ -331,12 +331,22 @@ static const struct kernfs_ops group_balancer_kf_single_ops = { .seq_show = group_balancer_seqfile_show, }; +static struct group_balancer_sched_domain *kn_parent_priv(struct kernfs_node *kn) +{ + /* + * The parent pointer is only valid within RCU section since it can be + * replaced + */ + guard(rcu)(); + return rcu_dereference(kn->__parent)->priv; +} + struct group_balancer_sched_domain *kernfs_to_gb_sd(struct kernfs_node *kn) { if (kernfs_type(kn) == KERNFS_DIR) return kn->priv; else - return kn->parent->priv; + return kn_parent_priv(kn); } struct group_balancer_sched_domain *group_balancer_kn_lock_live(struct kernfs_node *kn) diff --git a/lib/bitmap.c b/lib/bitmap.c index ddb31015e38ae23baf7b44b8aea0b5c53b3989aa..4bd5fb263b1d963551fa1c714511e1ee245f84f3 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -359,6 +359,13 @@ unsigned int __bitmap_weight_and(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_weight_and); +unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + return BITMAP_WEIGHT(bitmap1[idx] & ~bitmap2[idx], bits); +} +EXPORT_SYMBOL(__bitmap_weight_andnot); + void __bitmap_set(unsigned long *map, unsigned int start, int len) { unsigned long *p = map + BIT_WORD(start); diff --git a/lib/cpumask.c b/lib/cpumask.c index 34335c1e72653dc97e55bab4c0fb9c9b260c315c..e77ee9d46f71bb0db882dbe85fa56e491a81c9c8 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -14,7 +14,7 @@ * @start: the start point of the iteration * @wrap: assume @n crossing @start terminates the iteration * - * Returns >= nr_cpu_ids on completion + * Return: >= nr_cpu_ids on completion * * Note: the @wrap argument is required for the start condition when * we cannot assume @start is set in @mask. @@ -48,8 +48,9 @@ EXPORT_SYMBOL(cpumask_next_wrap); * @node: memory node from which to allocate or %NUMA_NO_NODE * * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is - * a nop returning a constant 1 (in ) - * Returns TRUE if memory allocation succeeded, FALSE otherwise. + * a nop returning a constant 1 (in ). + * + * Return: TRUE if memory allocation succeeded, FALSE otherwise. * * In addition, mask will be NULL if this fails. Note that gcc is * usually smart enough to know that mask can never be NULL if @@ -115,7 +116,7 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask) * @i: index number * @node: local numa_node * - * Returns online CPU according to a numa aware policy; local cpus are returned + * Return: online CPU according to a numa aware policy; local cpus are returned * first, followed by non-local ones, then it wraps around. * * For those who wants to enumerate all CPUs based on their NUMA distances, @@ -163,7 +164,7 @@ static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); * Iterated calls using the same srcp1 and srcp2 will be distributed within * their intersection. * - * Returns >= nr_cpu_ids if the intersection is empty. + * Return: >= nr_cpu_ids if the intersection is empty. */ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) @@ -182,6 +183,12 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, } EXPORT_SYMBOL(cpumask_any_and_distribute); +/** + * cpumask_any_distribute - Return an arbitrary cpu from srcp + * @srcp: &cpumask for selection + * + * Return: >= nr_cpu_ids if the intersection is empty. + */ unsigned int cpumask_any_distribute(const struct cpumask *srcp) { unsigned int next, prev; diff --git a/lib/find_bit.c b/lib/find_bit.c index dacadd904250fb500a3a68f19c1fc3e3df696554..79c36041694a8ea73d2c61deb0ccc6515c8929ee 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -116,6 +116,17 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, EXPORT_SYMBOL(_find_first_and_bit); #endif +/* + * Find the first bit set in 1st memory region and unset in 2nd. + */ +unsigned long _find_first_andnot_bit(const unsigned long *addr1, + const unsigned long *addr2, + unsigned long size) +{ + return FIND_FIRST_BIT(addr1[idx] & ~addr2[idx], /* nop */, size); +} +EXPORT_SYMBOL(_find_first_andnot_bit); + /* * Find the first set bit in three memory regions. */ diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index d4a99d98ec774558a6c36b4d4ff71d1fe61e88d4..8a8e4d51f298da57a1e40489af1eb94a48b7c2db 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3567,10 +3567,13 @@ static int selinux_kernfs_init_security(struct kernfs_node *kn_dir, newsid = tsec->create_sid; } else { u16 secclass = inode_mode_to_security_class(kn->mode); + const char *kn_name; struct qstr q; - q.name = kn->name; - q.hash_len = hashlen_string(kn_dir, kn->name); + /* kn is fresh, can't be renamed, name goes not away */ + kn_name = rcu_dereference_check(kn->name, true); + q.name = kn_name; + q.hash_len = hashlen_string(kn_dir, kn_name); rc = security_transition_sid(tsec->sid, parent_sid, secclass, &q, diff --git a/tools/testing/selftests/bpf/progs/profiler.inc.h b/tools/testing/selftests/bpf/progs/profiler.inc.h index f799d87e8700276bc8c2b9e74f21ae0e7285b151..cf5d64af07cf491b28403d6ded159b97408373e5 100644 --- a/tools/testing/selftests/bpf/progs/profiler.inc.h +++ b/tools/testing/selftests/bpf/progs/profiler.inc.h @@ -225,7 +225,7 @@ static INLINE void* read_full_cgroup_path(struct kernfs_node* cgroup_node, barrier_var(filepart_length); payload += filepart_length; } - cgroup_node = BPF_CORE_READ(cgroup_node, parent); + cgroup_node = BPF_CORE_READ(cgroup_node, __parent); } return payload; }